You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by "eldenmoon (via GitHub)" <gi...@apache.org> on 2026/05/12 19:26:20 UTC
[PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
eldenmoon opened a new pull request, #63192:
URL: https://github.com/apache/doris/pull/63192
### What problem does this PR solve?
Issue Number: N/A
Related PR: N/A
Problem Summary: Doris could not read Iceberg v3 VARIANT columns from Parquet files. This change maps Iceberg VARIANT to Doris VARIANT, decodes unshredded VARIANT metadata/value encoding, reads shredded typed_value columns, and prunes shredded Parquet leaf columns for accessed variant paths with a profile observable.
### Release note
Support reading Iceberg v3 VARIANT Parquet columns, including shredded typed_value column pruning.
### Check List (For Author)
- Test: Regression test / Unit Test / Manual test
- Regression test: ./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant
- Unit Test: JAVA_TOOL_OPTIONS=-Djdk.attach.allowAttachSelf=true ./run-fe-ut.sh --coverage --run org.apache.doris.nereids.rules.rewrite.VariantPruningLogicTest
- Manual test: ./build.sh --be --fe; build-support/clang-format.sh; build-support/check-format.sh; git diff --check
- Behavior changed: Yes. Doris can read Iceberg v3 VARIANT Parquet columns and avoid reading unneeded shredded typed_value/value leaves for selected variant subpaths.
- Does this need documentation: No
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3231490657
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,498 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+Status validate_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+void append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ json->append("null");
+ return;
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, dictionary_size + 1, offset_size,
+ "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ result->dictionary.emplace_back(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
+ }
+ return Status::OK();
+}
+
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next) {
+ switch (primitive_header) {
+ case 0:
+ json->append("null");
+ *next = ptr;
+ return Status::OK();
+ case 1:
+ json->append("true");
+ *next = ptr;
+ return Status::OK();
+ case 2:
+ json->append("false");
+ *next = ptr;
+ return Status::OK();
+ case 3:
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "int8 value"));
+ json->append(std::to_string(static_cast<int8_t>(*ptr)));
+ *next = ptr + 1;
+ return Status::OK();
+ case 4:
+ RETURN_IF_ERROR(require_available(ptr, end, 2, "int16 value"));
+ json->append(std::to_string(read_signed_le(ptr, 2)));
+ *next = ptr + 2;
+ return Status::OK();
+ case 5:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "int32 value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 6:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "int64 value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 7: {
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "double value"));
+ uint64_t bits = read_unsigned_le(ptr, 8);
+ double value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 8;
+ return Status::OK();
+ }
+ case 8:
+ case 9:
+ case 10: {
+ int value_size = primitive_header == 8 ? 4 : primitive_header == 9 ? 8 : 16;
+ RETURN_IF_ERROR(require_available(ptr, end, 1 + value_size, "decimal value"));
+ int scale = static_cast<int8_t>(*ptr++);
+ __int128 unscaled = 0;
+ if (value_size == 16) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ if ((ptr[15] & 0x80) != 0) {
+ unscaled = -static_cast<__int128>((~unsigned_value) + 1);
+ } else {
+ unscaled = static_cast<__int128>(unsigned_value);
+ }
+ } else {
+ unscaled = read_signed_le(ptr, value_size);
+ }
+ append_decimal_json(unscaled, scale, json);
+ *next = ptr + value_size;
+ return Status::OK();
+ }
+ case 11:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "date value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 12:
+ case 13:
+ case 17:
+ case 18:
+ case 19:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "time or timestamp value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 14: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "float value"));
+ uint32_t bits = static_cast<uint32_t>(read_unsigned_le(ptr, 4));
+ float value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 4;
+ return Status::OK();
+ }
+ case 15: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "binary value"));
+ return Status::NotSupported("Parquet VARIANT binary primitive is not supported");
+ }
+ case 16: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary or string length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "string value"));
+ append_json_string(
+ std::string_view(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size)),
+ json);
+ *next = ptr + size;
+ return Status::OK();
+ }
+ case 20:
+ RETURN_IF_ERROR(require_available(ptr, end, 16, "uuid value"));
+ append_uuid_json(ptr, json);
+ *next = ptr + 16;
+ return Status::OK();
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT primitive header {}",
+ primitive_header);
+ }
+}
+
+Status decode_object(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, std::string* json, const uint8_t** next) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int field_id_size = ((value_header >> 2) & 0x03) + 1;
+ int num_elements_size = (value_header & 0x10) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "object element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(
+ require_available_entries(ptr, end, num_elements, field_id_size, "object field ids"));
+ std::vector<uint64_t> field_ids(num_elements);
+ for (uint64_t i = 0; i < num_elements; ++i) {
+ field_ids[i] = read_unsigned_le(ptr, field_id_size);
+ ptr += field_id_size;
+ if (field_ids[i] >= metadata.dictionary.size()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field id {}", field_ids[i]);
+ }
+ }
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "object field offsets"));
+ std::vector<uint64_t> field_offsets(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ uint64_t total_size = field_offsets.back();
+ const uint8_t* fields = ptr;
+ RETURN_IF_ERROR(require_available(fields, end, total_size, "object field values"));
+ RETURN_IF_ERROR(validate_field_offsets(field_offsets, total_size, "object"));
Review Comment:
This rejects valid Parquet VARIANT objects. The VARIANT encoding requires field IDs to be sorted by key, but it explicitly says the encoded value entries do not need to be stored in that same order, so object `field_offset` entries may be non-monotonic. With offsets like `[4, 2, 0, 6]` for sorted keys whose values are laid out in a different byte order, `validate_field_offsets()` returns corruption here and the `child_end = field_offsets[i + 1]` logic below cannot decode the value. Arrays need monotonic offsets, but objects need separate handling: validate each offset is within `total_size`, decode from each offset using the full object payload (or compute bounds from the set of offsets), and verify decoded value boundaries without requiring adjacent offsets to increase. Please add a unit/regression case for an object whose values are stored out of field-id order.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437461741
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4438047383
<details>
<summary>TPC-DS: <b>Total hot run time: 170698 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit bcda6244eb29152d7533075da24e9c6b6dc1990f, data reload: false
query5 4344 661 516 516
query6 338 230 198 198
query7 4249 520 314 314
query8 332 231 224 224
query9 8863 4017 4066 4017
query10 460 358 302 302
query11 5794 2409 2178 2178
query12 182 138 129 129
query13 1303 603 451 451
query14 6046 5386 5050 5050
query14_1 4389 4379 4402 4379
query15 213 237 181 181
query16 979 444 431 431
query17 1122 766 620 620
query18 2434 479 373 373
query19 215 202 157 157
query20 139 132 129 129
query21 212 143 116 116
query22 13620 13552 13407 13407
query23 17156 16331 16599 16331
query23_1 16431 16340 16221 16221
query24 7858 1824 1404 1404
query24_1 1427 1446 1402 1402
query25 610 563 474 474
query26 1310 310 172 172
query27 2752 617 338 338
query28 4421 1974 1970 1970
query29 1012 638 550 550
query30 302 237 197 197
query31 1115 1070 932 932
query32 90 73 72 72
query33 528 357 297 297
query34 1168 1152 652 652
query35 769 787 680 680
query36 1312 1327 1155 1155
query37 155 100 90 90
query38 3187 3186 3091 3091
query39 919 943 886 886
query39_1 879 873 884 873
query40 238 162 141 141
query41 70 67 65 65
query42 109 111 109 109
query43 319 326 279 279
query44
query45 212 203 193 193
query46 1074 1197 710 710
query47 2275 2265 2173 2173
query48 415 416 288 288
query49 644 550 454 454
query50 726 281 218 218
query51 4310 4240 4275 4240
query52 107 112 95 95
query53 256 282 206 206
query54 324 283 276 276
query55 95 93 85 85
query56 362 294 311 294
query57 1392 1356 1311 1311
query58 296 265 264 264
query59 1508 1576 1420 1420
query60 355 342 327 327
query61 167 159 179 159
query62 654 617 557 557
query63 256 191 203 191
query64 2448 818 674 674
query65
query66 1759 515 389 389
query67 30041 29961 29448 29448
query68
query69 448 337 311 311
query70 1013 992 957 957
query71 310 274 271 271
query72 3040 2713 2413 2413
query73 870 736 438 438
query74 5095 4901 4724 4724
query75 2806 2640 2334 2334
query76 2301 1126 736 736
query77 425 447 345 345
query78 12920 12999 12397 12397
query79 1504 933 711 711
query80 1363 576 491 491
query81 516 276 235 235
query82 1268 158 122 122
query83 346 277 246 246
query84 272 139 114 114
query85 920 511 473 473
query86 453 332 308 308
query87 3452 3370 3200 3200
query88 3515 2660 2631 2631
query89 446 374 338 338
query90 1906 180 176 176
query91 178 165 149 149
query92 78 76 72 72
query93 973 952 551 551
query94 733 359 288 288
query95 657 456 334 334
query96 1023 747 336 336
query97 2690 2691 2585 2585
query98 240 233 225 225
query99 1113 1104 975 975
Total cold run time: 255566 ms
Total hot run time: 170698 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434643198
# FE UT Coverage Report
Increment line coverage ` 25.00% (1/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_e9e3bfd819ba9d8ccea3f9b57abb35147175a7e8/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_e9e3bfd819ba9d8ccea3f9b57abb35147175a7e8/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434628992
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4436900635
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440467427
<details>
<summary>TPC-DS: <b>Total hot run time: 169139 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 918c87c5454a4a3edef6fd9d50ea1e15b8d35c98, data reload: false
query5 4313 637 514 514
query6 340 219 208 208
query7 4220 580 303 303
query8 315 228 238 228
query9 8825 3988 4005 3988
query10 446 344 296 296
query11 5822 2341 2216 2216
query12 187 128 126 126
query13 1354 616 427 427
query14 6000 5359 5054 5054
query14_1 4361 4381 4340 4340
query15 211 204 185 185
query16 1004 459 357 357
query17 963 778 641 641
query18 2457 502 372 372
query19 223 214 166 166
query20 142 138 131 131
query21 219 146 124 124
query22 13621 13468 13397 13397
query23 17269 16335 15956 15956
query23_1 16166 16170 16190 16170
query24 7473 1773 1335 1335
query24_1 1356 1357 1363 1357
query25 595 531 464 464
query26 1306 309 170 170
query27 2721 618 342 342
query28 4439 1997 2021 1997
query29 1010 675 541 541
query30 314 241 202 202
query31 1145 1064 942 942
query32 99 79 75 75
query33 547 349 340 340
query34 1155 1123 633 633
query35 749 793 661 661
query36 1338 1323 1131 1131
query37 153 105 94 94
query38 3194 3124 3033 3033
query39 922 927 890 890
query39_1 891 876 864 864
query40 235 157 140 140
query41 70 64 62 62
query42 107 108 106 106
query43 321 324 282 282
query44
query45 209 198 194 194
query46 1093 1170 738 738
query47 2264 2305 2148 2148
query48 408 414 285 285
query49 649 533 422 422
query50 735 289 226 226
query51 4331 4269 4239 4239
query52 106 107 95 95
query53 254 280 200 200
query54 318 267 261 261
query55 93 91 88 88
query56 309 311 298 298
query57 1372 1389 1284 1284
query58 289 265 261 261
query59 1531 1586 1394 1394
query60 345 332 322 322
query61 162 157 160 157
query62 670 621 560 560
query63 237 209 200 200
query64 2449 842 690 690
query65
query66 1748 527 392 392
query67 30038 29397 29871 29397
query68
query69 473 333 297 297
query70 1029 992 1023 992
query71 308 275 269 269
query72 2938 2673 2403 2403
query73 796 809 428 428
query74 5080 4899 4724 4724
query75 2758 2659 2334 2334
query76 2294 1142 769 769
query77 416 435 340 340
query78 12889 12976 12296 12296
query79 1465 930 701 701
query80 1356 571 507 507
query81 522 274 239 239
query82 1123 157 128 128
query83 340 275 240 240
query84 318 143 110 110
query85 907 509 446 446
query86 459 322 326 322
query87 3420 3341 3180 3180
query88 3519 2644 2657 2644
query89 442 379 333 333
query90 1925 176 178 176
query91 178 168 137 137
query92 77 76 73 73
query93 1053 946 568 568
query94 716 350 286 286
query95 665 479 351 351
query96 1036 762 341 341
query97 2681 2690 2546 2546
query98 235 235 223 223
query99 1107 1126 985 985
Total cold run time: 253527 ms
Total hot run time: 169139 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440930135
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3234810189
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,541 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+void append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ json->append("null");
+ return;
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, dictionary_size + 1, offset_size,
+ "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ result->dictionary.emplace_back(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
Review Comment:
`decode_object()` now rejects duplicate or out-of-order field ids, but duplicate field names can still enter through the metadata dictionary itself. A malformed metadata dictionary like `["a", "a"]` with object field ids `[0, 1]` passes the increasing-id check and is serialized as JSON with duplicate `"a"` keys instead of corruption. Please reject duplicate dictionary entries during metadata decode and add a unit case for duplicate metadata names; this is distinct from the existing object field-id duplicate thread because the field ids are strictly increasing here.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +119,856 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return true;
Review Comment:
This classifies any typed struct with binary user fields named `metadata` and `value` as a Parquet VARIANT wrapper even when there is no `typed_value` child. A valid typed-only object like `typed_value.obj { metadata: string, value: string }` should decode as `{"obj":{"metadata":...,"value":...}}`, but `typed_struct_to_json()` routes `obj` through `variant_to_json()`, interpreting the user bytes as structural VARIANT metadata/value and either corrupting or dropping the object. Please require an actual shredded-wrapper shape here instead of `has_metadata && has_value` alone, and add coverage for typed object fields named `metadata` and `value`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3229734781
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1091,368 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
Review Comment:
Shredded `FLOAT`/`DOUBLE` values are emitted with `Field::to_debug_string()` and then parsed as JSON. For Parquet-valid NaN/Inf values this can produce non-JSON tokens, while the unshredded decoder explicitly normalizes non-finite floating values to JSON `null` via `append_floating_json()`. That makes the same VARIANT value fail or decode differently depending on whether it is stored in `value` or `typed_value`. Please handle non-finite float/double values consistently in the shredded typed-value path and add a regression/unit case.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1091,368 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
+ case TYPE_TIMEV2:
+ json->append(std::to_string(field.get<TYPE_TIMEV2>()));
+ return Status::OK();
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY: {
+ std::string value = field.to_debug_string(type->get_scale());
+ append_json_string(value, json);
+ return Status::OK();
+ }
+ case TYPE_ARRAY:
+ return _array_field_to_json(field_schema, field,
+ assert_cast<const DataTypeArray*>(type.get()), json);
+ case TYPE_STRUCT:
+ return _struct_field_to_json(field_schema, field, json);
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+Status VariantColumnReader::_array_field_to_json(const FieldSchema& field_schema,
+ const Field& field,
+ const DataTypeArray* array_type,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_ARRAY>();
+ const FieldSchema* element_schema =
+ field_schema.children.empty() ? nullptr : field_schema.children.data();
+ FieldSchema synthetic_element;
+ if (element_schema == nullptr) {
+ synthetic_element.data_type = array_type->get_nested_type();
+ element_schema = &synthetic_element;
+ }
+ json->push_back('[');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(
+ _field_to_json(*element_schema, values[i], &element_json, &element_present));
+ json->append(element_present ? element_json : "null");
+ }
+ json->push_back(']');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_struct_field_to_json(const FieldSchema& field_schema,
+ const Field& field, std::string* json) const {
+ const auto& values = field.get<TYPE_STRUCT>();
+ json->push_back('{');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ const FieldSchema& child_schema = field_schema.children[i];
+ append_json_string(child_schema.name, json);
+ json->push_back(':');
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(_field_to_json(child_schema, values[i], &child_json, &child_present));
+ json->append(child_present ? child_json : "null");
+ }
+ json->push_back('}');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_typed_value_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ *present = true;
+ return _typed_struct_to_json(typed_value_field, field, metadata, json);
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ *present = true;
+ return _typed_array_to_json(typed_value_field, field, metadata, json);
+ }
+ return _field_to_json(typed_value_field, field, json, present);
+}
+
+Status VariantColumnReader::_typed_struct_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_STRUCT>();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(_variant_to_json(typed_value_field.children[i], values[i], &metadata,
+ &child_json, &child_present));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ first = false;
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ }
+ json->push_back('}');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_typed_array_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_ARRAY>();
+ const FieldSchema* element_schema =
+ typed_value_field.children.empty() ? nullptr : typed_value_field.children.data();
+ json->push_back('[');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ if (element_schema != nullptr && find_child_idx(*element_schema, "value") >= 0) {
+ RETURN_IF_ERROR(_variant_to_json(*element_schema, values[i], &metadata, &element_json,
Review Comment:
This only recognizes shredded array elements when the element group contains a `value` child, but the Parquet Variant Shredding spec allows array `element` to omit `value` when elements are fully shredded as a specific `typed_value` type. For a valid schema like `typed_value (LIST) -> element { optional binary typed_value (STRING) }`, this branch falls through to `_field_to_json(*element_schema, ...)` and reconstructs each element as an object such as `{"typed_value":"x"}` instead of `"x"`. Please treat an element group with either `value` or `typed_value` as a shredded variant element and add coverage for the value-omitted array-element layout.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4438650069
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440012600
<details>
<summary>TPC-DS: <b>Total hot run time: 170013 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 1beb1746c3e4fbef9cb87898dc837d92107b692a, data reload: false
query5 4309 653 509 509
query6 331 213 199 199
query7 4214 531 300 300
query8 325 221 212 212
query9 8828 4043 4002 4002
query10 471 343 307 307
query11 5939 2389 2213 2213
query12 189 130 127 127
query13 1334 645 437 437
query14 6781 5335 5075 5075
query14_1 4343 4353 4324 4324
query15 217 210 182 182
query16 1007 456 436 436
query17 1241 761 641 641
query18 2728 485 360 360
query19 257 211 169 169
query20 138 135 130 130
query21 218 139 119 119
query22 13559 13551 13335 13335
query23 17336 16305 15976 15976
query23_1 16129 16219 16180 16180
query24 7538 1778 1355 1355
query24_1 1334 1350 1361 1350
query25 589 520 467 467
query26 1303 313 180 180
query27 2695 591 334 334
query28 4384 1992 1948 1948
query29 1033 655 534 534
query30 307 242 198 198
query31 1140 1056 943 943
query32 91 76 74 74
query33 556 365 306 306
query34 1160 1106 653 653
query35 758 812 676 676
query36 1358 1349 1191 1191
query37 151 103 90 90
query38 3221 3155 3026 3026
query39 946 942 888 888
query39_1 878 864 858 858
query40 233 153 139 139
query41 65 64 62 62
query42 109 105 109 105
query43 314 326 285 285
query44
query45 203 203 211 203
query46 1051 1188 724 724
query47 2366 2357 2210 2210
query48 398 412 293 293
query49 638 524 432 432
query50 689 285 210 210
query51 4278 4225 4193 4193
query52 105 103 100 100
query53 256 272 202 202
query54 308 268 248 248
query55 94 93 85 85
query56 295 299 295 295
query57 1442 1426 1316 1316
query58 293 263 265 263
query59 1536 1642 1446 1446
query60 350 333 316 316
query61 176 158 160 158
query62 674 635 544 544
query63 235 190 201 190
query64 2320 816 664 664
query65
query66 1705 526 394 394
query67 29934 29981 29806 29806
query68
query69 458 334 306 306
query70 1050 946 968 946
query71 302 274 269 269
query72 3038 2619 2444 2444
query73 864 757 441 441
query74 5067 4936 4742 4742
query75 2781 2680 2310 2310
query76 2319 1150 781 781
query77 423 435 349 349
query78 12965 12967 12398 12398
query79 1483 956 726 726
query80 1391 585 494 494
query81 522 279 237 237
query82 1247 158 123 123
query83 351 275 245 245
query84 265 141 113 113
query85 901 505 454 454
query86 441 345 349 345
query87 3412 3357 3214 3214
query88 3493 2628 2652 2628
query89 445 378 338 338
query90 1934 175 185 175
query91 197 163 136 136
query92 78 79 73 73
query93 956 952 562 562
query94 708 336 296 296
query95 661 371 352 352
query96 1007 794 346 346
query97 2713 2690 2547 2547
query98 244 231 229 229
query99 1127 1101 969 969
Total cold run time: 255181 ms
Total hot run time: 170013 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441989139
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4457399095
<details>
<summary>TPC-H: <b>Total hot run time: 29887 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit e42bb8169a7ca02afa39b1f28690edfdbe5bfa75, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17694 3935 3900 3900
q2 q3 10726 886 626 626
q4 4658 464 348 348
q5 7467 1356 1118 1118
q6 185 165 135 135
q7 925 949 749 749
q8 9330 1437 1327 1327
q9 6157 5557 5395 5395
q10 6298 2080 1851 1851
q11 490 266 259 259
q12 661 414 293 293
q13 18188 3359 2782 2782
q14 288 284 264 264
q15 q16 912 864 790 790
q17 911 954 737 737
q18 6545 5727 5683 5683
q19 1185 1272 1099 1099
q20 522 396 267 267
q21 4664 2292 1935 1935
q22 473 381 329 329
Total cold run time: 98279 ms
Total hot run time: 29887 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4640 4527 4610 4527
q2 q3 4683 4801 4225 4225
q4 2145 2248 1398 1398
q5 4997 5059 5312 5059
q6 195 171 136 136
q7 2062 1813 1609 1609
q8 3457 3093 3103 3093
q9 8445 8855 8435 8435
q10 4512 4509 4279 4279
q11 600 436 416 416
q12 721 753 512 512
q13 3115 3523 2912 2912
q14 301 310 276 276
q15 q16 792 788 705 705
q17 1352 1325 1253 1253
q18 8026 7186 7178 7178
q19 1178 1170 1157 1157
q20 2239 2223 2010 2010
q21 6166 5538 4841 4841
q22 530 478 400 400
Total cold run time: 60156 ms
Total hot run time: 54421 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4458063352
# BE Regression && UT Coverage Report
Increment line coverage `77.96% (1507/1933)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_e42bb8169a7ca02afa39b1f28690edfdbe5bfa75_merge/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_e42bb8169a7ca02afa39b1f28690edfdbe5bfa75_merge/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 73.65% (27910/37895) |
| Line Coverage | 57.60% (302795/525652) |
| Region Coverage | 54.82% (252931/461371) |
| Branch Coverage | 56.37% (109613/194445) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467147560
<details>
<summary>TPC-H: <b>Total hot run time: 30974 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 996c67ff18c29901aa8e182abd2ad228ed3b3611, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17755 3854 3827 3827
q2 q3 10801 1375 814 814
q4 4688 474 345 345
q5 7578 2258 2103 2103
q6 235 174 142 142
q7 945 749 632 632
q8 9357 1646 1674 1646
q9 5166 4860 4868 4860
q10 6403 2074 1802 1802
q11 419 265 246 246
q12 629 421 293 293
q13 18116 3406 2762 2762
q14 260 252 235 235
q15 q16 814 761 712 712
q17 984 877 917 877
q18 7187 5714 5476 5476
q19 1363 1289 1021 1021
q20 506 396 265 265
q21 6438 2835 2600 2600
q22 462 378 316 316
Total cold run time: 100106 ms
Total hot run time: 30974 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4651 4625 4560 4560
q2 q3 4954 5256 4588 4588
q4 2127 2217 1396 1396
q5 4691 4837 4681 4681
q6 227 174 129 129
q7 1948 1707 1521 1521
q8 2344 2087 2065 2065
q9 7779 7418 7231 7231
q10 4480 4399 4004 4004
q11 521 387 364 364
q12 702 729 504 504
q13 3041 3588 2752 2752
q14 282 269 264 264
q15 q16 689 690 608 608
q17 1260 1229 1220 1220
q18 7285 6855 6755 6755
q19 1122 1081 1114 1081
q20 2208 2235 1912 1912
q21 5289 4588 4571 4571
q22 549 469 416 416
Total cold run time: 56149 ms
Total hot run time: 50622 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467476609
<details>
<summary>TPC-DS: <b>Total hot run time: 169503 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 1ffd4832e5e0db6a9a2557137111c28714971f23, data reload: false
query5 4309 661 515 515
query6 337 216 199 199
query7 4279 558 312 312
query8 319 234 217 217
query9 8834 3967 3971 3967
query10 470 331 293 293
query11 5818 2385 2180 2180
query12 184 138 130 130
query13 1291 607 436 436
query14 5946 5330 5064 5064
query14_1 4324 4335 4310 4310
query15 212 202 184 184
query16 1011 445 445 445
query17 1154 776 619 619
query18 2713 483 353 353
query19 223 205 177 177
query20 143 133 129 129
query21 221 149 121 121
query22 13591 13516 13557 13516
query23 17161 16334 16088 16088
query23_1 16233 16140 16189 16140
query24 7441 1760 1306 1306
query24_1 1288 1329 1282 1282
query25 588 502 437 437
query26 1303 370 170 170
query27 2666 552 332 332
query28 4422 1934 1907 1907
query29 948 622 493 493
query30 323 254 200 200
query31 1128 1064 941 941
query32 85 75 73 73
query33 540 342 285 285
query34 1152 1124 631 631
query35 767 772 690 690
query36 1331 1310 1177 1177
query37 158 103 90 90
query38 3201 3124 3045 3045
query39 931 935 907 907
query39_1 867 880 879 879
query40 223 145 123 123
query41 69 63 64 63
query42 109 110 110 110
query43 319 322 290 290
query44
query45 215 200 188 188
query46 1082 1154 734 734
query47 2340 2342 2191 2191
query48 382 394 291 291
query49 648 500 390 390
query50 1003 336 253 253
query51 4360 4347 4245 4245
query52 106 105 94 94
query53 257 275 200 200
query54 306 287 263 263
query55 93 88 85 85
query56 298 306 306 306
query57 1424 1400 1322 1322
query58 303 267 264 264
query59 1533 1604 1403 1403
query60 323 315 307 307
query61 161 154 153 153
query62 668 616 569 569
query63 249 198 216 198
query64 2365 803 682 682
query65
query66 1653 465 374 374
query67 29894 29904 29749 29749
query68
query69 473 347 306 306
query70 1092 1040 1083 1040
query71 321 285 282 282
query72 2952 2682 2374 2374
query73 831 783 426 426
query74 5036 4904 4689 4689
query75 2667 2604 2286 2286
query76 2281 1110 773 773
query77 402 413 332 332
query78 12161 12145 11646 11646
query79 1413 1009 732 732
query80 666 529 454 454
query81 445 275 240 240
query82 1419 155 124 124
query83 357 283 250 250
query84 264 145 113 113
query85 873 539 452 452
query86 397 347 330 330
query87 3420 3421 3213 3213
query88 3538 2652 2681 2652
query89 475 389 346 346
query90 1877 184 176 176
query91 177 171 143 143
query92 79 76 76 76
query93 1466 1373 824 824
query94 532 353 315 315
query95 684 385 437 385
query96 1073 795 348 348
query97 2716 2707 2577 2577
query98 237 224 234 224
query99 1117 1123 978 978
Total cold run time: 252617 ms
Total hot run time: 169503 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253401198
##########
be/src/format/parquet/parquet_nested_column_utils.cpp:
##########
@@ -0,0 +1,529 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <cctype>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "core/data_type/data_type_nullable.h"
+#include "format/parquet/schema_desc.h"
+
+namespace doris {
+namespace {
+
+enum class NestedPathMode {
+ NAME,
+ FIELD_ID,
+};
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_variant_typed_child_by_key(const FieldSchema& field_schema,
+ std::string_view key, NestedPathMode mode) {
+ if (const auto* child = find_child_by_exact_name(field_schema, key)) {
+ return child;
+ }
+ if (mode == NestedPathMode::NAME) {
+ return nullptr;
+ }
+ for (const auto& child : field_schema.children) {
+ if (child.field_id >= 0 && key == std::to_string(child.field_id)) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+using VariantPathMap = std::unordered_map<std::string, std::vector<std::vector<std::string>>>;
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return true;
+ }
+ if (typed_value == nullptr) {
+ return false;
+ }
+ const auto type = remove_nullable(typed_value->data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+bool add_shredded_variant_field_value(const FieldSchema& shredded_field,
+ std::set<uint64_t>& column_ids) {
+ if (const auto* value = find_child_by_structural_name(shredded_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ return true;
+ }
+ return false;
+}
+
+bool is_variant_array_subscript(std::string_view path) {
+ return !path.empty() &&
+ std::all_of(path.begin(), path.end(), [](unsigned char c) { return std::isdigit(c); });
+}
+
+bool is_terminal_variant_meta_component(std::string_view path) {
+ return path == "NULL" || path == "OFFSET";
+}
+
+const std::vector<std::string>& effective_variant_path(const std::vector<std::string>& raw_path,
+ std::vector<std::string>& stripped_path) {
+ if (!raw_path.empty() && is_terminal_variant_meta_component(raw_path.back())) {
+ stripped_path.assign(raw_path.begin(), raw_path.end() - 1);
+ return stripped_path;
+ }
+ return raw_path;
+}
+
+bool contains_inherited_metadata_value(const FieldSchema& field_schema) {
+ if (is_shredded_variant_field(field_schema) &&
+ find_child_by_structural_name(field_schema, "value") != nullptr) {
+ return true;
+ }
+ return std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_inherited_metadata_value(child); });
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode);
+
+VariantColumnIdExtractionResult extract_typed_value_path(const FieldSchema& typed_value,
+ const std::vector<std::string>& path,
+ std::set<uint64_t>& column_ids,
+ NestedPathMode mode) {
+ VariantColumnIdExtractionResult result;
+ const auto typed_value_type = remove_nullable(typed_value.data_type);
+ if (typed_value_type->get_primitive_type() != TYPE_STRUCT) {
+ result = extract_variant_typed_nested_column_ids(typed_value, {path}, column_ids, mode);
+ } else if (const auto* typed_child =
+ find_variant_typed_child_by_key(typed_value, path[0], mode)) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata = contains_inherited_metadata_value(*typed_child);
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ result = extract_variant_typed_nested_column_ids(*typed_child, child_paths, column_ids,
+ mode);
+ }
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(typed_value.get_column_id());
+ }
+ return result;
+}
+
+void add_variant_typed_path(PrimitiveType field_type, const FieldSchema& field_schema,
+ const std::vector<std::string>& path,
+ VariantColumnIdExtractionResult* result, std::set<uint64_t>& column_ids,
+ VariantPathMap* child_paths) {
+ if (path.empty()) {
+ add_column_id_range(field_schema, column_ids);
+ result->has_child_columns = true;
+ result->needs_metadata |= contains_inherited_metadata_value(field_schema);
+ return;
+ }
+
+ const bool is_list = field_type == PrimitiveType::TYPE_ARRAY;
+ const bool is_map = field_type == PrimitiveType::TYPE_MAP;
+ std::vector<std::string> remaining;
+ std::string child_key;
+ if (is_list) {
+ child_key = "*";
+ if (!is_variant_array_subscript(path[0])) {
+ remaining.assign(path.begin(), path.end());
+ } else if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ } else if (is_map) {
+ (*child_paths)["KEYS"].emplace_back();
+ child_key = "VALUES";
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ } else {
+ child_key = path[0];
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ }
+ (*child_paths)[child_key].push_back(std::move(remaining));
+}
+
+std::string variant_typed_child_key(PrimitiveType field_type, const FieldSchema& field_schema,
+ uint64_t child_index) {
+ if (field_type == PrimitiveType::TYPE_ARRAY) {
+ return "*";
+ }
+ if (field_type == PrimitiveType::TYPE_MAP) {
+ if (child_index == 0) {
+ return "KEYS";
+ }
+ return child_index == 1 ? "VALUES" : "";
+ }
+ return field_schema.children[child_index].name;
+}
+
+void append_variant_child_paths(const VariantPathMap& paths_by_name, const std::string& key,
+ std::vector<std::vector<std::string>>& child_paths) {
+ auto child_paths_it = paths_by_name.find(key);
+ if (child_paths_it != paths_by_name.end()) {
+ child_paths.insert(child_paths.end(), child_paths_it->second.begin(),
+ child_paths_it->second.end());
+ }
+}
+
+std::vector<std::vector<std::string>> collect_variant_typed_child_paths(
+ const VariantPathMap& paths_by_name, const FieldSchema& child, const std::string& child_key,
+ bool is_list, bool is_map, NestedPathMode mode) {
+ std::vector<std::vector<std::string>> child_paths;
+ append_variant_child_paths(paths_by_name, child_key, child_paths);
+ if (!is_list && !is_map && mode == NestedPathMode::FIELD_ID && child.field_id >= 0) {
+ const std::string field_id_key = std::to_string(child.field_id);
+ if (field_id_key != child_key) {
+ append_variant_child_paths(paths_by_name, field_id_key, child_paths);
+ }
+ }
+ return child_paths;
+}
+
+void extract_variant_typed_child_column_ids(
+ const FieldSchema& child, const std::vector<std::vector<std::string>>& child_paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode,
+ VariantColumnIdExtractionResult* result) {
+ const bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+ if (needs_full_child) {
+ add_column_id_range(child, column_ids);
+ result->has_child_columns = true;
+ result->needs_metadata |= contains_inherited_metadata_value(child);
+ return;
+ }
+
+ auto child_result =
+ extract_variant_typed_nested_column_ids(child, child_paths, column_ids, mode);
+ result->has_child_columns |= child_result.has_child_columns;
+ result->needs_metadata |= child_result.needs_metadata;
+}
+
+VariantColumnIdExtractionResult extract_shredded_variant_field_ids(
+ const FieldSchema& shredded_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode) {
+ const auto* typed_value = find_child_by_structural_name(shredded_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& raw_path : paths) {
+ std::vector<std::string> stripped_path;
+ const auto& path = effective_variant_path(raw_path, stripped_path);
+ if (path.empty()) {
+ add_column_id_range(shredded_field, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata |= contains_inherited_metadata_value(shredded_field);
+ continue;
+ }
+
+ VariantColumnIdExtractionResult typed_result;
+ if (typed_value != nullptr) {
+ typed_result = extract_typed_value_path(*typed_value, path, column_ids, mode);
+ result.needs_metadata |= typed_result.needs_metadata;
+ }
+ if (!typed_result.has_child_columns) {
Review Comment:
This only keeps the field-level residual `value` when no typed child was selected at all. For a shredded field that has both `value` and a nested typed subpath, e.g. `typed_value.metric { value, typed_value: struct<x:int> }`, an access to `v['metric']['x']` selects `metric.typed_value.x` and then skips `metric.value`. But rows whose `metric` value is stored in the field-level residual object can still contain `x`; those rows need `metric.value` plus inherited metadata to evaluate the same subpath. The new regression currently asserts that `v.typed_value.metric.value` is not read for `metric.x`, so this mixed typed/residual case would silently return NULL/missing for residual rows. This is distinct from the existing deeper-path thread where the typed child was absent and the fallback selected the wrong top-level `v.value`; here the typed child exists, but the required field-level residual is pruned away. Please keep the shredded field residual whenever a deeper subpath is requested
under a field that has a residual `value`, and add coverage where some rows satisfy `metric.x` through `metric.typed_value.x` and others through `metric.value`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468341474
<details>
<summary>TPC-DS: <b>Total hot run time: 168942 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 5c8d26e78b19d9103df693de698dbb57f3a65ab1, data reload: false
query5 4302 646 501 501
query6 336 223 199 199
query7 4258 590 289 289
query8 320 222 216 216
query9 8856 4076 4058 4058
query10 456 337 293 293
query11 5768 2353 2273 2273
query12 184 126 123 123
query13 1253 603 432 432
query14 5995 5322 5019 5019
query14_1 4367 4352 4371 4352
query15 221 203 180 180
query16 1031 456 451 451
query17 1156 748 602 602
query18 2631 494 371 371
query19 219 209 178 178
query20 149 131 129 129
query21 217 155 117 117
query22 13593 13500 13424 13424
query23 17228 16470 16035 16035
query23_1 16140 16180 16179 16179
query24 7451 1762 1294 1294
query24_1 1284 1306 1283 1283
query25 545 465 416 416
query26 1303 303 174 174
query27 2696 547 339 339
query28 4429 1936 1934 1934
query29 1027 616 488 488
query30 297 236 199 199
query31 1143 1063 933 933
query32 89 77 73 73
query33 532 349 285 285
query34 1169 1108 636 636
query35 774 785 658 658
query36 1354 1368 1202 1202
query37 158 100 87 87
query38 3242 3120 3065 3065
query39 935 922 903 903
query39_1 886 865 861 861
query40 228 151 128 128
query41 68 63 63 63
query42 112 110 108 108
query43 321 321 288 288
query44
query45 215 196 191 191
query46 1025 1191 722 722
query47 2357 2330 2227 2227
query48 395 391 294 294
query49 621 515 400 400
query50 967 360 244 244
query51 4309 4270 4208 4208
query52 104 104 91 91
query53 259 276 200 200
query54 309 261 246 246
query55 91 92 89 89
query56 301 297 289 289
query57 1422 1401 1325 1325
query58 331 272 271 271
query59 1565 1625 1392 1392
query60 323 314 305 305
query61 153 155 157 155
query62 663 616 579 579
query63 244 204 205 204
query64 2409 813 631 631
query65
query66 1675 466 341 341
query67 29325 29893 29705 29705
query68
query69 456 383 297 297
query70 1050 966 992 966
query71 308 272 263 263
query72 3062 2636 2368 2368
query73 806 735 411 411
query74 5060 4870 4702 4702
query75 2685 2582 2253 2253
query76 2290 1146 746 746
query77 402 400 324 324
query78 12023 12171 11622 11622
query79 1276 1027 735 735
query80 584 549 479 479
query81 475 278 245 245
query82 235 153 116 116
query83 271 282 249 249
query84 262 140 109 109
query85 926 579 503 503
query86 382 333 325 325
query87 3366 3344 3252 3252
query88 3480 2646 2617 2617
query89 428 388 336 336
query90 2179 179 178 178
query91 175 167 134 134
query92 78 75 72 72
query93 1497 1398 843 843
query94 551 363 299 299
query95 725 372 348 348
query96 1060 779 342 342
query97 2703 2681 2546 2546
query98 233 233 242 233
query99 1130 1109 957 957
Total cold run time: 250984 ms
Total hot run time: 168942 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468242926
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468464066
<details>
<summary>TPC-H: <b>Total hot run time: 31601 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 7cbca3a936be42996b737e56c61ce4c1658ffacc, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17665 3963 3857 3857
q2 q3 10825 1403 837 837
q4 4686 472 336 336
q5 7558 2230 2159 2159
q6 244 173 138 138
q7 951 751 629 629
q8 9373 1664 1536 1536
q9 5202 4876 4883 4876
q10 6387 2091 1789 1789
q11 447 276 245 245
q12 671 419 299 299
q13 18219 3402 2857 2857
q14 267 254 245 245
q15 q16 814 764 711 711
q17 940 1008 987 987
q18 6961 5640 5600 5600
q19 1300 1298 1129 1129
q20 505 443 380 380
q21 6258 2840 2668 2668
q22 460 382 323 323
Total cold run time: 99733 ms
Total hot run time: 31601 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4676 4831 4534 4534
q2 q3 4921 5150 4646 4646
q4 2121 2188 1383 1383
q5 4703 4261 4246 4246
q6 221 164 121 121
q7 1760 1646 1384 1384
q8 2235 1946 1907 1907
q9 7243 7281 7171 7171
q10 4513 4396 3966 3966
q11 530 382 349 349
q12 710 724 501 501
q13 3090 3375 2823 2823
q14 278 291 248 248
q15 q16 678 690 610 610
q17 1258 1249 1233 1233
q18 7186 6946 6819 6819
q19 1118 1111 1085 1085
q20 2216 2201 1954 1954
q21 5319 4581 4557 4557
q22 527 460 408 408
Total cold run time: 55303 ms
Total hot run time: 49945 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470025714
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466708522
<details>
<summary>TPC-H: <b>Total hot run time: 30786 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 81c4010234456ca39d939382b19f1af624aeec55, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17700 3870 3885 3870
q2 q3 10810 1376 802 802
q4 4703 469 351 351
q5 7783 2214 2118 2118
q6 354 175 134 134
q7 965 755 655 655
q8 9366 1691 1550 1550
q9 6997 4901 4855 4855
q10 6451 2112 1774 1774
q11 433 278 247 247
q12 695 417 290 290
q13 18210 3367 2738 2738
q14 269 256 231 231
q15 q16 813 772 712 712
q17 926 902 944 902
q18 6704 5817 5418 5418
q19 1169 1298 1140 1140
q20 538 400 264 264
q21 5820 2632 2436 2436
q22 418 353 299 299
Total cold run time: 101124 ms
Total hot run time: 30786 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4189 4085 4077 4077
q2 q3 4485 4919 4288 4288
q4 2068 2216 1382 1382
q5 4367 4245 4257 4245
q6 233 172 128 128
q7 2163 1915 1705 1705
q8 2419 2125 2031 2031
q9 7814 7703 7757 7703
q10 4567 4487 4072 4072
q11 553 427 371 371
q12 705 826 616 616
q13 3367 3587 2984 2984
q14 289 286 275 275
q15 q16 712 730 640 640
q17 1325 1314 1281 1281
q18 7947 7410 6849 6849
q19 1126 1093 1085 1085
q20 2204 2204 1922 1922
q21 5246 4574 4488 4488
q22 512 471 399 399
Total cold run time: 56291 ms
Total hot run time: 50541 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3252904278
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,576 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+__int128 read_signed_int128_le(const uint8_t* ptr) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ static constexpr unsigned __int128 sign_bit = static_cast<unsigned __int128>(1) << 127;
+ if ((unsigned_value & sign_bit) == 0) {
+ return static_cast<__int128>(unsigned_value);
+ }
+ static constexpr __int128 signed_half_range = static_cast<__int128>(1) << 126;
+ return (static_cast<__int128>(unsigned_value & (sign_bit - 1)) - signed_half_range) -
+ signed_half_range;
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+bool variant_string_less(std::string_view lhs, std::string_view rhs) {
+ return std::lexicographical_compare(
+ lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](char left, char right) {
+ return static_cast<unsigned char>(left) < static_cast<unsigned char>(right);
+ });
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+Status append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point value is not supported");
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+ return Status::OK();
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ const bool sorted_strings = (header & 0x10) != 0;
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, dictionary_size + 1, offset_size,
+ "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+ if (offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ if (ptr + offsets.back() != end) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary bytes");
+ }
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ std::string entry(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
Review Comment:
`decode_metadata()` copies dictionary entries directly from the metadata byte buffer without validating that they are valid UTF-8, and `append_json_string()` later emits those bytes into JSON unchanged. The Parquet VARIANT string/dictionary grammar requires UTF-8, so a malformed metadata dictionary key containing invalid bytes (for example `0xff`) currently passes metadata decoding and produces invalid JSON or a downstream parse-dependent result instead of reporting corruption. Please validate UTF-8 for metadata dictionary keys (and string values decoded through the same JSON path) and add malformed-string coverage.
##########
be/src/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp:
##########
@@ -18,21 +18,313 @@
#include "format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <iostream>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
+#include "core/data_type/data_type_nullable.h"
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return true;
+ }
+ if (typed_value == nullptr) {
+ return false;
+ }
+ const auto type = remove_nullable(typed_value->data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+bool add_shredded_variant_field_value(const FieldSchema& shredded_field,
+ std::set<uint64_t>& column_ids) {
+ if (const auto* value = find_child_by_structural_name(shredded_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ return true;
+ }
+ return false;
+}
+
+bool is_variant_array_subscript(std::string_view path) {
+ return !path.empty() &&
+ std::all_of(path.begin(), path.end(), [](unsigned char c) { return std::isdigit(c); });
+}
+
+bool contains_inherited_metadata_value(const FieldSchema& field_schema) {
+ if (is_shredded_variant_field(field_schema) &&
+ find_child_by_structural_name(field_schema, "value") != nullptr) {
+ return true;
+ }
+ return std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_inherited_metadata_value(child); });
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+VariantColumnIdExtractionResult extract_shredded_variant_field_ids(
+ const FieldSchema& shredded_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_structural_name(shredded_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(shredded_field, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata |= contains_inherited_metadata_value(shredded_field);
+ continue;
+ }
+
+ bool has_selected_columns = add_shredded_variant_field_value(shredded_field, column_ids);
+ result.needs_metadata |= has_selected_columns;
+ if (typed_value != nullptr) {
+ const auto typed_value_type = remove_nullable(typed_value->data_type);
+ if (typed_value_type->get_primitive_type() != TYPE_STRUCT) {
+ auto child_result =
+ extract_variant_typed_nested_column_ids(*typed_value, {path}, column_ids);
+ if (child_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ result.needs_metadata |= child_result.needs_metadata;
+ has_selected_columns = true;
+ }
+ } else if (const auto* typed_child = find_child_by_exact_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ result.needs_metadata |= contains_inherited_metadata_value(*typed_child);
+ column_ids.insert(typed_value->get_column_id());
+ has_selected_columns = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ auto child_result = extract_variant_typed_nested_column_ids(
+ *typed_child, child_paths, column_ids);
+ if (child_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ result.needs_metadata |= child_result.needs_metadata;
+ has_selected_columns = true;
+ }
+ }
+ }
+ }
+ result.has_child_columns |= has_selected_columns;
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(shredded_field.get_column_id());
+ }
+ return result;
+}
+
+VariantColumnIdExtractionResult extract_variant_nested_column_ids(
+ const FieldSchema& variant_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_structural_name(variant_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ result.has_child_columns = true;
+ continue;
+ }
+
+ VariantColumnIdExtractionResult typed_result;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_exact_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ typed_result.has_child_columns = true;
+ typed_result.needs_metadata = contains_inherited_metadata_value(*typed_child);
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
Review Comment:
The Iceberg duplicate has the same META-suffix issue as the Hive/local helper. A path such as `[metric, NULL]` for `v['metric'] IS NULL` recurses into scalar `typed_value.metric`, selects no typed column, and then falls back to top-level `v.value`, even though the null check needs the typed leaf's definition levels/null map. Please handle terminal `NULL`/`OFFSET` suffixes as selecting the current field here as well and cover the Iceberg path.
##########
be/src/format/table/hive/hive_parquet_nested_column_utils.cpp:
##########
@@ -18,20 +18,312 @@
#include "format/table/hive/hive_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
+#include "core/data_type/data_type_nullable.h"
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return true;
+ }
+ if (typed_value == nullptr) {
+ return false;
+ }
+ const auto type = remove_nullable(typed_value->data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+bool add_shredded_variant_field_value(const FieldSchema& shredded_field,
+ std::set<uint64_t>& column_ids) {
+ if (const auto* value = find_child_by_structural_name(shredded_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ return true;
+ }
+ return false;
+}
+
+bool is_variant_array_subscript(std::string_view path) {
+ return !path.empty() &&
+ std::all_of(path.begin(), path.end(), [](unsigned char c) { return std::isdigit(c); });
+}
+
+bool contains_inherited_metadata_value(const FieldSchema& field_schema) {
+ if (is_shredded_variant_field(field_schema) &&
+ find_child_by_structural_name(field_schema, "value") != nullptr) {
+ return true;
+ }
+ return std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_inherited_metadata_value(child); });
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+VariantColumnIdExtractionResult extract_shredded_variant_field_ids(
+ const FieldSchema& shredded_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_structural_name(shredded_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(shredded_field, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata |= contains_inherited_metadata_value(shredded_field);
+ continue;
+ }
+
+ bool has_selected_columns = add_shredded_variant_field_value(shredded_field, column_ids);
+ result.needs_metadata |= has_selected_columns;
+ if (typed_value != nullptr) {
+ const auto typed_value_type = remove_nullable(typed_value->data_type);
+ if (typed_value_type->get_primitive_type() != TYPE_STRUCT) {
+ auto child_result =
+ extract_variant_typed_nested_column_ids(*typed_value, {path}, column_ids);
+ if (child_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ result.needs_metadata |= child_result.needs_metadata;
+ has_selected_columns = true;
+ }
+ } else if (const auto* typed_child = find_child_by_exact_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ result.needs_metadata |= contains_inherited_metadata_value(*typed_child);
+ column_ids.insert(typed_value->get_column_id());
+ has_selected_columns = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ auto child_result = extract_variant_typed_nested_column_ids(
+ *typed_child, child_paths, column_ids);
+ if (child_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ result.needs_metadata |= child_result.needs_metadata;
+ has_selected_columns = true;
+ }
+ }
+ }
+ }
+ result.has_child_columns |= has_selected_columns;
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(shredded_field.get_column_id());
+ }
+ return result;
+}
+
+VariantColumnIdExtractionResult extract_variant_nested_column_ids(
+ const FieldSchema& variant_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_structural_name(variant_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ result.has_child_columns = true;
+ continue;
+ }
+
+ VariantColumnIdExtractionResult typed_result;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_exact_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ typed_result.has_child_columns = true;
+ typed_result.needs_metadata = contains_inherited_metadata_value(*typed_child);
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
Review Comment:
This VARIANT pruning path treats metadata suffixes such as `NULL` as ordinary child names. For a predicate/access like `v['metric'] IS NULL`, FE can pass a path like `[metric, NULL]`; since `path.size() != 1`, the code recurses into the scalar `typed_value.metric`, finds no child named `NULL`, and then falls back to top-level `v.value` below. That omits `v.typed_value.metric` even though the null check needs its definition levels/null map, so Hive/local Parquet can evaluate the predicate from the wrong residual path or miss the typed shard entirely. Please treat terminal `NULL`/`OFFSET` META suffixes as selecting the current field before recursing, and add coverage for a VARIANT subpath null predicate.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +123,1126 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper,
+ bool allow_value_only_wrapper = false) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr || allow_value_only_wrapper;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ JsonBinaryValue empty_object;
+ RETURN_IF_ERROR(empty_object.from_json_string("{}"));
+ (*values)[path] = FieldWithDataType {.field = Field::create_field<TYPE_JSONB>(JsonbField(
+ empty_object.value(), empty_object.size())),
+ .base_scalar_type_id = TYPE_JSONB};
+ return Status::OK();
+}
+
+static bool is_empty_object_marker(const FieldWithDataType& value) {
+ if (value.field.get_type() != TYPE_JSONB) {
+ return false;
+ }
+ const auto& jsonb = value.field.get<TYPE_JSONB>();
+ const JsonbDocument* document = nullptr;
+ Status st =
+ JsonbDocument::checkAndCreateDocument(jsonb.get_value(), jsonb.get_size(), &document);
+ if (!st.ok() || document == nullptr || document->getValue() == nullptr ||
+ !document->getValue()->isObject()) {
+ return false;
+ }
+ return document->getValue()->unpack<ObjectVal>()->numElem() == 0;
+}
+
+static Status collect_empty_object_markers(const rapidjson::Value& value, PathInDataBuilder* path,
+ VariantMap* values) {
+ if (!value.IsObject()) {
+ return Status::OK();
+ }
+ if (value.MemberCount() == 0) {
+ return insert_empty_object_marker(path->build(), values);
+ }
+ for (auto it = value.MemberBegin(); it != value.MemberEnd(); ++it) {
+ if (it->value.IsObject()) {
+ path->append(std::string_view(it->name.GetString(), it->name.GetStringLength()), false);
+ RETURN_IF_ERROR(collect_empty_object_markers(it->value, path, values));
+ path->pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+static Status add_empty_object_markers_from_json(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ if (json.find("{}") == std::string::npos) {
+ return Status::OK();
+ }
+ rapidjson::Document document;
+ document.Parse(json.data(), json.size());
+ if (document.HasParseError()) {
+ return Status::Corruption("Invalid Parquet VARIANT decoded JSON");
+ }
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ return collect_empty_object_markers(document, &path, values);
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (!parsed.is_null()) {
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ }
+ RETURN_IF_ERROR(add_empty_object_markers_from_json(json, prefix, values));
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static bool path_has_prefix(const PathInData& path, const PathInData& prefix) {
+ const auto& parts = path.get_parts();
+ const auto& prefix_parts = prefix.get_parts();
+ if (parts.size() < prefix_parts.size()) {
+ return false;
+ }
+ for (size_t i = 0; i < prefix_parts.size(); ++i) {
+ if (parts[i] != prefix_parts[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool has_descendant_path(const VariantMap& values, const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ return std::ranges::any_of(values, [&](const auto& entry) {
+ const auto& path = entry.first;
+ return path.get_parts().size() > prefix_size && path_has_prefix(path, prefix);
+ });
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* values,
+ const VariantMap& shadowing_values) {
+ for (auto it = values->begin(); it != values->end();) {
+ if (is_empty_object_marker(it->second) &&
+ (has_descendant_path(*values, it->first) ||
+ has_descendant_path(shadowing_values, it->first))) {
+ it = values->erase(it);
+ continue;
+ }
+ ++it;
+ }
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* value_values,
+ VariantMap* typed_values) {
+ erase_shadowed_empty_object_markers(value_values, *typed_values);
+ erase_shadowed_empty_object_markers(typed_values, *value_values);
+}
+
+static Status check_no_shredded_value_typed_duplicates(const VariantMap& value_values,
+ const VariantMap& typed_values,
+ const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ for (const auto& value_entry : value_values) {
+ const auto& value_path = value_entry.first;
+ if (!path_has_prefix(value_path, prefix)) {
+ continue;
+ }
+ if (value_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(value_entry.second) &&
+ !has_descendant_path(typed_values, value_path)) {
+ continue;
+ }
+ if (!typed_values.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT residual value conflicts with typed_value at path {}",
+ value_path.get_path());
+ }
+ continue;
+ }
+ for (const auto& typed_entry : typed_values) {
+ const auto& typed_path = typed_entry.first;
+ if (!path_has_prefix(typed_path, prefix)) {
+ continue;
+ }
+ if (typed_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(typed_entry.second) &&
+ !has_descendant_path(value_values, typed_path)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ if (value_path.get_parts()[prefix_size] == typed_path.get_parts()[prefix_size]) {
+ if (value_path == typed_path && is_empty_object_marker(value_entry.second) &&
+ is_empty_object_marker(typed_entry.second)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+static bool has_direct_typed_parent_null(const std::vector<const NullMap*>& null_maps, size_t row) {
+ return std::ranges::any_of(null_maps, [&](const NullMap* null_map) {
+ DCHECK_LT(row, null_map->size());
+ return (*null_map)[row];
+ });
+}
+
+static void insert_direct_typed_leaf_range(const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps,
+ IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ nullable_leaf.get_nested_column().insert_range_from(*value_column, start, rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static bool is_temporal_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static DataTypePtr direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return std::make_shared<DataTypeInt64>();
+ }
+ return type;
+}
+
+static bool contains_temporal_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_temporal_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static int64_t direct_temporal_variant_value(PrimitiveType type, const IColumn& column,
+ size_t row) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ return static_cast<int64_t>(
+ std::llround(assert_cast<const ColumnTimeV2&>(column).get_data()[row]));
+ case TYPE_DATE:
+ return variant_date_value(assert_cast<const ColumnDate&>(column).get_data()[row]);
+ case TYPE_DATETIME:
+ return variant_datetime_value(assert_cast<const ColumnDateTime&>(column).get_data()[row]);
+ case TYPE_DATEV2:
+ return variant_date_value(assert_cast<const ColumnDateV2&>(column).get_data()[row]);
+ case TYPE_DATETIMEV2:
+ return variant_datetime_value(assert_cast<const ColumnDateTimeV2&>(column).get_data()[row]);
+ case TYPE_TIMESTAMPTZ:
+ return variant_datetime_value(
+ assert_cast<const ColumnTimeStampTz&>(column).get_data()[row]);
+ default:
+ DORIS_CHECK(false);
+ return 0;
+ }
+}
+
+static void insert_direct_typed_temporal_leaf_range(
+ PrimitiveType type, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnInt64&>(nullable_leaf.get_nested_column()).get_data();
+ data.reserve(data.size() + rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ data.push_back(direct_temporal_variant_value(type, *value_column, row));
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ return true;
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return !contains_temporal_variant_leaf_type(array_type->get_nested_type()) &&
+ is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool has_selected_direct_typed_leaf(const FieldSchema& field_schema,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return false;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ return has_selected_direct_typed_leaf(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_direct_typed_leaf(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool can_use_direct_typed_only_value(const FieldSchema& variant_field,
+ const std::set<uint64_t>& column_ids) {
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ return (value_idx < 0 || !has_selected_column(variant_field.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ has_selected_direct_typed_leaf(variant_field.children[typed_value_idx], false,
+ column_ids) &&
+ can_direct_read_typed_value(variant_field.children[typed_value_idx], false, column_ids);
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static void fill_variant_leaf_type_info(const DataTypePtr& data_type, FieldWithDataType* value) {
+ auto leaf_type = remove_nullable(data_type);
+ while (leaf_type->get_primitive_type() == TYPE_ARRAY) {
+ leaf_type = remove_nullable(
+ assert_cast<const DataTypeArray*>(leaf_type.get())->get_nested_type());
+ }
+ if (is_decimal(leaf_type->get_primitive_type())) {
+ value->precision = leaf_type->get_precision();
+ value->scale = leaf_type->get_scale();
+ }
+}
+
+template <PrimitiveType Primitive>
+static Status fill_floating_point_variant_field(const Field& field, FieldWithDataType* value) {
+ const auto typed_value = field.get<Primitive>();
+ if (!std::isfinite(typed_value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point typed_value is not supported");
+ }
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ fill_variant_leaf_type_info(type, value);
+ return Status::OK();
+ case TYPE_FLOAT:
+ return fill_floating_point_variant_field<TYPE_FLOAT>(field, value);
+ case TYPE_DOUBLE:
+ return fill_floating_point_variant_field<TYPE_DOUBLE>(field, value);
+ case TYPE_TIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>())));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATE:
+ value->field = Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATE>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIME:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIME>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATEV2:
+ value->field =
+ Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIMEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_VARBINARY:
+ return Status::NotSupported("Parquet VARIANT binary typed_value is not supported");
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present);
+
+static Status serialize_field_to_json(const DataTypePtr& data_type, const Field& field,
+ std::string* json) {
+ MutableColumnPtr column = data_type->create_column();
+ column->insert(field);
+
+ auto json_column = ColumnString::create();
+ VectorBufferWriter writer(*json_column);
+ auto serde = data_type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ RETURN_IF_ERROR(serde->serialize_one_cell_to_json(*column, 0, writer, options));
+ writer.commit();
+ *json = json_column->get_data_at(0).to_string();
+ return Status::OK();
+}
+
+static Status scalar_typed_value_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(field_schema, field, &value, present));
+ if (!*present) {
+ return Status::OK();
+ }
+ if (value.field.is_null()) {
+ *json = "null";
+ return Status::OK();
+ }
+
+ DataTypePtr json_type;
+ if (value.base_scalar_type_id != PrimitiveType::INVALID_TYPE) {
+ json_type = DataTypeFactory::instance().create_data_type(value.base_scalar_type_id, false,
+ value.precision, value.scale);
+ } else {
+ json_type = remove_nullable(field_schema.data_type);
+ }
+ return serialize_field_to_json(json_type, value.field, json);
+}
+
+static Status resolve_variant_metadata(const FieldSchema& variant_field, const Struct& fields,
+ const std::string* inherited_metadata, std::string* metadata,
+ bool* has_metadata) {
+ *has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ *metadata = *inherited_metadata;
+ *has_metadata = true;
+ }
+
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], metadata, &metadata_present));
+ *has_metadata = metadata_present;
+ }
+ return Status::OK();
+}
+
+static Status variant_typed_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, std::string* typed_json,
+ bool* typed_present) {
+ *typed_present = false;
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ if (typed_value_idx < 0) {
+ return Status::OK();
+ }
+ return typed_value_to_json(variant_field.children[typed_value_idx], fields[typed_value_idx],
+ metadata, typed_json, typed_present);
+}
+
+static Status variant_residual_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, bool has_metadata,
+ std::string* value_json, bool* value_present) {
+ *value_present = false;
+ const int value_idx = find_child_idx(variant_field, "value");
+ if (value_idx < 0) {
+ return Status::OK();
+ }
+
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, value_present));
+ if (!*value_present) {
+ return Status::OK();
+ }
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ return parquet::decode_variant_to_json(StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), value_json);
+}
+
+static Status merge_variant_value_and_typed_json(const std::string& value_json,
+ const std::string& typed_json, std::string* json) {
+ VariantMap value_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, PathInData(), &value_values));
+ VariantMap typed_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(typed_json, PathInData(), &typed_values));
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto root_value = value_values.find(PathInData());
+ if (root_value != value_values.end() && !is_empty_object_marker(root_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, PathInData()));
+ value_values.merge(std::move(typed_values));
+ return variant_map_to_json(std::move(value_values), json);
+}
+
+static Status variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ std::string metadata;
+ bool has_metadata = false;
+ RETURN_IF_ERROR(resolve_variant_metadata(variant_field, fields, inherited_metadata, &metadata,
+ &has_metadata));
+
+ std::string typed_json;
+ bool typed_present = false;
+ RETURN_IF_ERROR(variant_typed_value_to_json(variant_field, fields, metadata, &typed_json,
+ &typed_present));
+
+ std::string value_json;
+ bool value_present = false;
+ RETURN_IF_ERROR(variant_residual_value_to_json(variant_field, fields, metadata, has_metadata,
+ &value_json, &value_present));
+
+ if (value_present && typed_present) {
+ RETURN_IF_ERROR(merge_variant_value_and_typed_json(value_json, typed_json, json));
+ *present = true;
+ return Status::OK();
+ }
+
+ if (typed_present) {
+ *json = std::move(typed_json);
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+
+ *present = false;
+ return Status::OK();
+}
+
+static Status shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json, bool* present,
+ bool allow_scalar_typed_value_only_wrapper) {
+ if (is_variant_wrapper_field(field_schema, allow_scalar_typed_value_only_wrapper, true)) {
+ return variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ return typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+static Status typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ json->clear();
+ json->push_back('[');
+ for (size_t i = 0; i < elements.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(element_schema, elements[i], metadata, &element_json,
+ &element_present, true));
+ if (element_present) {
+ json->append(element_json);
+ } else {
+ return Status::Corruption("Parquet VARIANT array typed_value element is missing");
+ }
+ }
+ json->push_back(']');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ json->clear();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(typed_value_field.children[i], fields[i], metadata,
+ &child_json, &child_present, false));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ first = false;
+ }
+ json->push_back('}');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ switch (typed_type->get_primitive_type()) {
+ case TYPE_STRUCT:
+ return typed_struct_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_ARRAY:
+ return typed_array_to_json(typed_value_field, field, metadata, json, present);
+ default:
+ return scalar_typed_value_to_json(typed_value_field, field, json, present);
+ }
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present);
+
+static Status variant_to_variant_map(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ VariantMap value_values;
+ bool value_present = false;
+ const PathInData current_path = path->build();
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ std::string value_json;
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, current_path, &value_values));
+ }
+ }
+
+ VariantMap typed_values;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_variant_map(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, path,
+ &typed_values, &typed_present));
+ }
+
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto current_value = value_values.find(current_path);
+ if (value_present && typed_present && current_value != value_values.end() &&
+ !is_empty_object_marker(current_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, current_path));
+ values->merge(std::move(value_values));
+ values->merge(std::move(typed_values));
+ *present = value_present || typed_present;
+ return Status::OK();
+}
+
+static Status shredded_field_to_variant_map(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (is_variant_wrapper_field(field_schema, false, true)) {
+ return variant_to_variant_map(field_schema, field, &metadata, path, values, present);
+ }
+ return typed_value_to_variant_map(field_schema, field, metadata, path, values, present);
+}
+
+static Status append_typed_field_to_variant_map(const FieldSchema& typed_value_field,
+ const Field& field, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static Status typed_array_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (is_direct_variant_leaf_type(typed_value_field.data_type)) {
+ return append_typed_field_to_variant_map(typed_value_field, field, path, values, present);
+ }
+
+ std::string value_json;
+ RETURN_IF_ERROR(typed_value_to_json(typed_value_field, field, metadata, &value_json, present));
+ if (*present) {
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, path->build(), values));
+ }
+ return Status::OK();
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& fields = field.get<TYPE_STRUCT>();
+ *present = true;
+ bool has_present_child = false;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ path->append(typed_value_field.children[i].name, false);
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_variant_map(typed_value_field.children[i], fields[i],
+ metadata, path, values, &child_present));
+ has_present_child |= child_present;
+ path->pop_back();
+ }
+ if (!has_present_child) {
+ RETURN_IF_ERROR(insert_empty_object_marker(path->build(), values));
+ }
+ return Status::OK();
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ return typed_array_to_variant_map(typed_value_field, field, metadata, path, values,
+ present);
+ }
+
+ return append_typed_field_to_variant_map(typed_value_field, field, path, values, present);
+}
+
+static Status append_direct_typed_column_to_batch(const FieldSchema& field_schema,
+ const IColumn& column, size_t start, size_t rows,
+ PathInDataBuilder* path, ColumnVariant* batch,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids,
+ std::vector<const NullMap*> parent_null_maps) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return Status::OK();
+ }
+
+ const IColumn* value_column = &column;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ parent_null_maps.push_back(&nullable_column->get_null_map_data());
+ value_column = &nullable_column->get_nested_column();
+ }
+
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ const auto& typed_struct = assert_cast<const ColumnStruct&>(*value_column);
+ return append_direct_typed_column_to_batch(
+ field_schema.children[typed_value_idx], typed_struct.get_column(typed_value_idx),
+ start, rows, path, batch, false, column_ids, parent_null_maps);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& struct_column = assert_cast<const ColumnStruct&>(*value_column);
Review Comment:
The direct typed-only path recursively adds only leaf subcolumns for structs, but it never emits the empty-object marker that the row-wise path adds when a present typed struct has no present children. For a typed-only row like `{'outer': {}}` where `outer` is a present struct and all its typed children are absent/null, row-wise reconstruction inserts an empty object marker at `outer`, while the direct path reaches this branch, adds no non-null leaf for `outer`, and materializes the row as missing/`{}` at the parent level. Please either make the direct path ineligible for nullable/all-null typed structs or add equivalent empty-object marker handling, with coverage for a typed-only nested empty object.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467350484
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467473843
# BE UT Coverage Report
Increment line coverage `75.72% (1987/2624)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/8468e1e4b59fc6dd19d32fdad2b2fbf0fbc402c4_8468e1e4b59fc6dd19d32fdad2b2fbf0fbc402c4/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/8468e1e4b59fc6dd19d32fdad2b2fbf0fbc402c4_8468e1e4b59fc6dd19d32fdad2b2fbf0fbc402c4/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.63% (20778/38745) |
| Line Coverage | 37.33% (196930/527543) |
| Region Coverage | 33.72% (154287/457492) |
| Branch Coverage | 34.79% (67504/194049) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467772127
# BE UT Coverage Report
Increment line coverage `75.96% (2193/2887)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/e1dce6934ce20d01059a46068509f8d202249881_e1dce6934ce20d01059a46068509f8d202249881/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/e1dce6934ce20d01059a46068509f8d202249881_e1dce6934ce20d01059a46068509f8d202249881/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.64% (20792/38760) |
| Line Coverage | 37.35% (197144/527791) |
| Region Coverage | 33.74% (154396/457673) |
| Branch Coverage | 34.81% (67594/194162) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253811118
##########
be/src/format/parquet/vparquet_reader.cpp:
##########
@@ -1339,8 +1469,7 @@ Status ParquetReader::_process_column_stat_filter(
}
const auto& file_col_name =
_table_info_node_ptr->children_file_column_name(slot->col_name());
- const FieldSchema* col_schema =
- _file_metadata->schema().get_column(file_col_name);
+ const FieldSchema* col_schema = parquet_file_schema().get_column(file_col_name);
int parquet_col_id = col_schema->physical_column_index;
auto meta_data = row_group.columns[parquet_col_id].meta_data;
stat->col_schema = col_schema;
Review Comment:
This row-group stats path now also sees top-level VARIANT predicates, but a VARIANT root is a group field with `physical_column_index == -1`. A predicate such as `v IS NULL` / `v IS NOT NULL` can enter `_process_column_stat_filter()`, map the slot to the top-level VARIANT schema, and index `row_group.columns[-1]` here before any guard. The page-index path below already returns false for `parquet_col_id < 0`; please add the same guard before reading min/max metadata and in the bloom-filter lambda as well, with coverage for a pushed predicate on a Parquet VARIANT column.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468997021
<details>
<summary>TPC-H: <b>Total hot run time: 30632 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 8f9fab0f853a0ff1c31e773d05417c655f93d626, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17789 3807 3826 3807
q2 q3 10988 1350 804 804
q4 4684 462 347 347
q5 7542 2193 2093 2093
q6 238 179 136 136
q7 942 786 638 638
q8 9376 1660 1585 1585
q9 6448 4826 4853 4826
q10 6446 2113 1747 1747
q11 447 279 245 245
q12 686 436 294 294
q13 18175 3472 2758 2758
q14 264 252 235 235
q15 q16 825 771 708 708
q17 1007 991 894 894
q18 6678 5725 5428 5428
q19 1246 1291 1095 1095
q20 537 416 264 264
q21 5610 2630 2424 2424
q22 429 352 304 304
Total cold run time: 100357 ms
Total hot run time: 30632 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4212 4129 4079 4079
q2 q3 4471 4870 4304 4304
q4 2079 2183 1386 1386
q5 4386 4241 4248 4241
q6 227 172 130 130
q7 1887 1855 1723 1723
q8 2464 2099 2063 2063
q9 7770 7727 7658 7658
q10 4510 4495 4141 4141
q11 579 417 384 384
q12 735 920 514 514
q13 3329 3581 2966 2966
q14 308 323 288 288
q15 q16 732 725 652 652
q17 1334 1289 1315 1289
q18 7775 7244 6870 6870
q19 1122 1063 1085 1063
q20 2214 2189 1932 1932
q21 5275 4615 4461 4461
q22 550 474 429 429
Total cold run time: 55959 ms
Total hot run time: 50573 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468999573
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469405763
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469805147
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467408337
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467588880
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468048622
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253689156
##########
regression-test/suites/external_table_p0/tvf/test_local_tvf_iceberg_variant.groovy:
##########
@@ -0,0 +1,444 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.net.InetAddress
+import java.net.NetworkInterface
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+import org.apache.doris.regression.action.ProfileAction
+
+suite("test_local_tvf_iceberg_variant", "p0,external") {
Review Comment:
This regression only scans the files through `local(...)`, so it bypasses the actual Iceberg table path changed by this PR. In particular, it does not exercise FE `SlotTypeReplacer.replaceIcebergAccessPathToId()` converting access paths to Iceberg field IDs, nor BE `IcebergParquetReader::_create_column_ids()` calling the Iceberg field-id pruning helper. A bug in field-id access-path serialization or Iceberg-specific column-id selection can therefore pass this suite while failing `SELECT v['...'] FROM iceberg_tbl`. Please add an end-to-end Iceberg external table regression (with nested VARIANT access and profile/read-column assertions where possible) that reads these VARIANT files through the Iceberg catalog/table path rather than only through local TVF.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468411652
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253780101
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java:
##########
@@ -657,13 +658,15 @@ private void replaceIcebergAccessPathToId(List<String> originPath, int index, Da
}
} else if (type instanceof StructType) {
for (Column child : column.getChildren()) {
- if (child.getName().equals(fieldName)) {
Review Comment:
Iceberg access-path conversion handles map values but not map keys. When a query accesses a nested map-key field, for example `map_keys(m)[1].a` on `MAP<STRUCT<a:int,b:int>, int>`, the collected path includes `KEYS, a`; this branch leaves `a` as a name instead of recursing into `column.getChildren().get(0)` and replacing it with the Iceberg key-field id. The BE Iceberg pruning path is then in field-id mode under `KEYS`, so the named `a` component will not match and the needed key subfield can be omitted. This is distinct from the existing MAP OFFSET/null reader and map-function argument collection threads because it is specifically the FE Iceberg field-id serialization for nested key paths. Please handle `ACCESS_MAP_KEYS` symmetrically with `ACCESS_MAP_VALUES` and add an Iceberg test for pruning a nested map-key field.
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,1156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cstring>
+#include <deque>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+#include "core/column/column_variant.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/value/jsonb_value.h"
+#include "exec/common/variant_util.h"
+
+namespace doris::parquet {
+
+std::string format_variant_uuid(const uint8_t* ptr) {
+ static constexpr char hex[] = "0123456789abcdef";
+ std::string uuid;
+ uuid.reserve(36);
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ uuid.push_back('-');
+ }
+ uuid.push_back(hex[ptr[i] >> 4]);
+ uuid.push_back(hex[ptr[i] & 0x0f]);
+ }
+ return uuid;
+}
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+struct VariantObjectLayout {
+ std::vector<uint64_t> field_ids;
+ std::vector<uint64_t> field_offsets;
+ std::vector<uint64_t> field_ends;
+ const uint8_t* fields = nullptr;
+ uint64_t total_size = 0;
+};
+
+struct VariantArrayLayout {
+ std::vector<uint64_t> field_offsets;
+ const uint8_t* fields = nullptr;
+ uint64_t total_size = 0;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+__int128 read_signed_int128_le(const uint8_t* ptr) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ static constexpr unsigned __int128 sign_bit = static_cast<unsigned __int128>(1) << 127;
+ if ((unsigned_value & sign_bit) == 0) {
+ return static_cast<__int128>(unsigned_value);
+ }
+ static constexpr __int128 signed_half_range = static_cast<__int128>(1) << 126;
+ return (static_cast<__int128>(unsigned_value & (sign_bit - 1)) - signed_half_range) -
+ signed_half_range;
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ if (size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+bool variant_string_less(std::string_view lhs, std::string_view rhs) {
+ return std::lexicographical_compare(
+ lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](char left, char right) {
+ return static_cast<unsigned char>(left) < static_cast<unsigned char>(right);
+ });
+}
+
+bool is_valid_utf8(std::string_view value) {
+ const auto* data = reinterpret_cast<const uint8_t*>(value.data());
+ const auto* end = data + value.size();
+ while (data < end) {
+ const uint8_t first = *data++;
+ if (first <= 0x7f) {
+ continue;
+ }
+
+ uint32_t code_point = 0;
+ size_t continuation_bytes = 0;
+ if (first >= 0xc2 && first <= 0xdf) {
+ code_point = first & 0x1f;
+ continuation_bytes = 1;
+ } else if (first >= 0xe0 && first <= 0xef) {
+ code_point = first & 0x0f;
+ continuation_bytes = 2;
+ } else if (first >= 0xf0 && first <= 0xf4) {
+ code_point = first & 0x07;
+ continuation_bytes = 3;
+ } else {
+ return false;
+ }
+
+ if (static_cast<size_t>(end - data) < continuation_bytes) {
+ return false;
+ }
+ for (size_t i = 0; i < continuation_bytes; ++i) {
+ const uint8_t byte = *data++;
+ if ((byte & 0xc0) != 0x80) {
+ return false;
+ }
+ code_point = (code_point << 6) | (byte & 0x3f);
+ }
+
+ if ((continuation_bytes == 2 && code_point < 0x800) ||
+ (continuation_bytes == 3 && code_point < 0x10000) ||
+ (code_point >= 0xd800 && code_point <= 0xdfff) || code_point > 0x10ffff) {
+ return false;
+ }
+ }
+ return true;
+}
+
+Status require_valid_utf8(std::string_view value, std::string_view context) {
+ if (!is_valid_utf8(value)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} UTF-8 string", context);
+ }
+ return Status::OK();
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json, bool escape_non_ascii = false) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20 || (escape_non_ascii && c >= 0x80)) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+Status append_floating_json(T value, std::string* json) {
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+ return Status::OK();
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next);
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ json->push_back('"');
+ json->append(format_variant_uuid(ptr));
+ json->push_back('"');
+}
+
+Status make_jsonb_field(std::string_view json, FieldWithDataType* value) {
+ JsonBinaryValue jsonb_value;
+ RETURN_IF_ERROR(jsonb_value.from_json_string(json.data(), json.size()));
+ value->field =
+ Field::create_field<TYPE_JSONB>(JsonbField(jsonb_value.value(), jsonb_value.size()));
+ value->base_scalar_type_id = TYPE_JSONB;
+ value->num_dimensions = 0;
+ value->precision = 0;
+ value->scale = 0;
+ return Status::OK();
+}
+
+std::string make_null_array_json(size_t elements) {
+ std::string json = "[";
+ for (size_t i = 0; i < elements; ++i) {
+ if (i != 0) {
+ json.push_back(',');
+ }
+ json.append("null");
+ }
+ json.push_back(']');
+ return json;
+}
+
+Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field("{}", &value));
+ (*values)[path] = std::move(value);
+ return Status::OK();
+}
+
+Status parse_json_to_variant_map(std::string_view json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (parsed.is_null()) {
+ (*values)[prefix] = FieldWithDataType {.field = Field()};
+ return Status::OK();
+ }
+
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ for (auto& [parsed_path, value] : parsed.get<TYPE_VARIANT>()) {
+ path.append(parsed_path.get_parts(), false);
+ (*values)[path.build()] = std::move(value);
+ for (size_t i = 0; i < parsed_path.get_parts().size(); ++i) {
+ path.pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+void fill_field_type_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+ value->precision = info.precision;
+ value->scale = info.scale;
+}
+
+template <PrimitiveType Primitive>
+void set_primitive_variant_field(const typename PrimitiveTypeTraits<Primitive>::CppType& data,
+ FieldWithDataType* value) {
+ value->field = Field::create_field<Primitive>(data);
+ fill_field_type_info(value);
+}
+
+Status read_decimal_primitive_field(uint8_t primitive_header, const uint8_t* ptr,
+ const uint8_t* end, FieldWithDataType* value,
+ const uint8_t** next) {
+ int value_size = 16;
+ if (primitive_header == 8) {
+ value_size = 4;
+ } else if (primitive_header == 9) {
+ value_size = 8;
+ }
+ RETURN_IF_ERROR(require_available(ptr, end, 1 + value_size, "decimal value"));
+ int scale = static_cast<int8_t>(*ptr++);
+ if (scale < 0 || scale > BeConsts::MAX_DECIMAL128_PRECISION) {
+ return Status::Corruption("Invalid Parquet VARIANT decimal scale {}", scale);
+ }
+
+ if (primitive_header == 8) {
+ set_primitive_variant_field<TYPE_DECIMAL32>(
+ Decimal32(static_cast<Int32>(read_signed_le(ptr, value_size))), value);
+ value->precision = BeConsts::MAX_DECIMAL32_PRECISION;
+ } else if (primitive_header == 9) {
+ set_primitive_variant_field<TYPE_DECIMAL64>(
+ Decimal64(static_cast<Int64>(read_signed_le(ptr, value_size))), value);
+ value->precision = BeConsts::MAX_DECIMAL64_PRECISION;
+ } else {
+ set_primitive_variant_field<TYPE_DECIMAL128I>(Decimal128V3(read_signed_int128_le(ptr)),
+ value);
+ value->precision = BeConsts::MAX_DECIMAL128_PRECISION;
+ }
+ value->scale = scale;
+ *next = ptr + value_size;
+ return Status::OK();
+}
+
+Status read_integral_primitive_field(uint8_t primitive_header, const uint8_t* ptr,
+ const uint8_t* end, FieldWithDataType* value,
+ const uint8_t** next) {
+ int value_size = 8;
+ if (primitive_header == 3) {
+ value_size = 1;
+ } else if (primitive_header == 4) {
+ value_size = 2;
+ } else if (primitive_header == 5 || primitive_header == 11) {
+ value_size = 4;
+ }
+ RETURN_IF_ERROR(require_available(ptr, end, value_size, "integer value"));
+ const auto data = static_cast<Int64>(read_signed_le(ptr, value_size));
+
+ switch (primitive_header) {
+ case 3:
+ set_primitive_variant_field<TYPE_TINYINT>(static_cast<Int8>(data), value);
+ break;
+ case 4:
+ set_primitive_variant_field<TYPE_SMALLINT>(static_cast<Int16>(data), value);
+ break;
+ case 5:
+ set_primitive_variant_field<TYPE_INT>(static_cast<Int32>(data), value);
+ break;
+ case 6:
+ case 11:
+ case 12:
+ case 13:
+ case 17:
+ set_primitive_variant_field<TYPE_BIGINT>(data, value);
+ break;
+ case 18:
+ case 19:
+ set_primitive_variant_field<TYPE_BIGINT>(data / 1000, value);
+ break;
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT primitive header {}",
+ primitive_header);
+ }
+ *next = ptr + value_size;
+ return Status::OK();
+}
+
+Status read_floating_primitive_field(uint8_t primitive_header, const uint8_t* ptr,
+ const uint8_t* end, FieldWithDataType* value,
+ const uint8_t** next) {
+ if (primitive_header == 14) {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "float value"));
+ auto bits = static_cast<uint32_t>(read_unsigned_le(ptr, 4));
+ float data;
+ std::memcpy(&data, &bits, sizeof(data));
+ set_primitive_variant_field<TYPE_FLOAT>(data, value);
+ *next = ptr + 4;
+ return Status::OK();
+ }
+
+ DCHECK_EQ(primitive_header, 7);
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "double value"));
+ uint64_t bits = read_unsigned_le(ptr, 8);
+ double data;
+ std::memcpy(&data, &bits, sizeof(data));
+ set_primitive_variant_field<TYPE_DOUBLE>(data, value);
+ *next = ptr + 8;
+ return Status::OK();
+}
+
+Status read_binary_primitive_field(const uint8_t* ptr, const uint8_t* end, FieldWithDataType* value,
+ std::deque<std::string>* string_values, const uint8_t** next) {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "binary value"));
+ string_values->emplace_back(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size));
+ value->field = Field::create_field<TYPE_VARBINARY>(StringView(string_values->back()));
+ fill_field_type_info(value);
+ *next = ptr + size;
+ return Status::OK();
+}
+
+Status read_string_primitive_field(const uint8_t* ptr, const uint8_t* end, FieldWithDataType* value,
+ const uint8_t** next) {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary or string length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "string value"));
+ std::string_view data(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size));
+ RETURN_IF_ERROR(require_valid_utf8(data, "string value"));
+ value->field = Field::create_field<TYPE_STRING>(String(data));
+ fill_field_type_info(value);
+ *next = ptr + size;
+ return Status::OK();
+}
+
+Status read_uuid_primitive_field(const uint8_t* ptr, const uint8_t* end, FieldWithDataType* value,
+ const uint8_t** next) {
+ RETURN_IF_ERROR(require_available(ptr, end, 16, "uuid value"));
+ value->field = Field::create_field<TYPE_STRING>(format_variant_uuid(ptr));
+ fill_field_type_info(value);
+ *next = ptr + 16;
+ return Status::OK();
+}
+
+Status read_array_layout(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ VariantArrayLayout* layout) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int num_elements_size = (value_header & 0x04) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "array element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "array field offsets"));
+ layout->field_offsets.resize(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ layout->field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ layout->total_size = layout->field_offsets.back();
+ layout->fields = ptr;
+ RETURN_IF_ERROR(
+ require_available(layout->fields, end, layout->total_size, "array field values"));
+ RETURN_IF_ERROR(
+ validate_array_field_offsets(layout->field_offsets, layout->total_size, "array"));
+ return Status::OK();
+}
+
+Status read_object_layout(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, VariantObjectLayout* layout) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int field_id_size = ((value_header >> 2) & 0x03) + 1;
+ int num_elements_size = (value_header & 0x10) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "object element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(
+ require_available_entries(ptr, end, num_elements, field_id_size, "object field ids"));
+ layout->field_ids.resize(num_elements);
+ for (uint64_t i = 0; i < num_elements; ++i) {
+ layout->field_ids[i] = read_unsigned_le(ptr, field_id_size);
+ ptr += field_id_size;
+ if (layout->field_ids[i] >= metadata.dictionary.size()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field id {}",
+ layout->field_ids[i]);
+ }
+ if (i > 0 && !variant_string_less(metadata.dictionary[layout->field_ids[i - 1]],
+ metadata.dictionary[layout->field_ids[i]])) {
+ return Status::Corruption("Invalid Parquet VARIANT object field names");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "object field offsets"));
+ layout->field_offsets.resize(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ layout->field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ layout->total_size = layout->field_offsets.back();
+ layout->fields = ptr;
+ RETURN_IF_ERROR(
+ require_available(layout->fields, end, layout->total_size, "object field values"));
+ RETURN_IF_ERROR(compute_object_field_ends(layout->field_offsets, layout->total_size,
+ &layout->field_ends));
+ return Status::OK();
+}
+
+Status decode_value_to_variant_map(const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, PathInDataBuilder* path,
+ VariantMap* values, std::deque<std::string>* string_values,
+ const uint8_t** next);
+
+Status decode_primitive_to_variant_map(uint8_t primitive_header, const uint8_t* ptr,
+ const uint8_t* end, const VariantMetadata&,
+ PathInDataBuilder* path, VariantMap* values,
+ std::deque<std::string>* string_values,
+ const uint8_t** next) {
+ FieldWithDataType value;
+ switch (primitive_header) {
+ case 0:
+ value.field = Field();
+ value.base_scalar_type_id = INVALID_TYPE;
+ *next = ptr;
+ break;
+ case 1:
+ set_primitive_variant_field<TYPE_BOOLEAN>(true, &value);
+ *next = ptr;
+ break;
+ case 2:
+ set_primitive_variant_field<TYPE_BOOLEAN>(false, &value);
+ *next = ptr;
+ break;
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 11:
+ case 12:
+ case 13:
+ case 17:
+ case 18:
+ case 19:
+ RETURN_IF_ERROR(read_integral_primitive_field(primitive_header, ptr, end, &value, next));
+ break;
+ case 7:
+ case 14:
+ RETURN_IF_ERROR(read_floating_primitive_field(primitive_header, ptr, end, &value, next));
+ break;
+ case 8:
+ case 9:
+ case 10:
+ RETURN_IF_ERROR(read_decimal_primitive_field(primitive_header, ptr, end, &value, next));
+ break;
+ case 15:
+ RETURN_IF_ERROR(read_binary_primitive_field(ptr, end, &value, string_values, next));
+ break;
+ case 16:
+ RETURN_IF_ERROR(read_string_primitive_field(ptr, end, &value, next));
+ break;
+ case 20:
+ RETURN_IF_ERROR(read_uuid_primitive_field(ptr, end, &value, next));
+ break;
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT primitive header {}",
+ primitive_header);
+ }
+ (*values)[path->build()] = std::move(value);
+ return Status::OK();
+}
+
+Status decode_object_to_variant_map(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, PathInDataBuilder* path,
+ VariantMap* values, std::deque<std::string>* string_values,
+ const uint8_t** next) {
+ VariantObjectLayout layout;
+ RETURN_IF_ERROR(read_object_layout(value_header, ptr, end, metadata, &layout));
+
+ if (layout.field_ids.empty()) {
+ RETURN_IF_ERROR(insert_empty_object_marker(path->build(), values));
+ }
+
+ for (uint64_t i = 0; i < layout.field_ids.size(); ++i) {
+ const uint8_t* child_begin = layout.fields + layout.field_offsets[i];
+ const uint8_t* child_end = layout.fields + layout.field_ends[i];
+ const uint8_t* child_next = nullptr;
+ path->append(metadata.dictionary[layout.field_ids[i]], false);
+ RETURN_IF_ERROR(decode_value_to_variant_map(child_begin, child_end, metadata, path, values,
+ string_values, &child_next));
+ path->pop_back();
+ if (child_next != child_end) {
+ return Status::Corruption("Invalid Parquet VARIANT object child value length");
+ }
+ }
+ *next = layout.fields + layout.total_size;
+ return Status::OK();
+}
+
+void move_variant_map_to_field(VariantMap&& element_values, FieldWithDataType* value) {
+ if (element_values.size() == 1 && element_values.begin()->first.empty()) {
+ *value = std::move(element_values.begin()->second);
+ return;
+ }
+ value->field = Field::create_field<TYPE_VARIANT>(std::move(element_values));
+ fill_field_type_info(value);
+}
+
+Status decode_array_element_to_field(const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, FieldWithDataType* value,
+ std::deque<std::string>* string_values, const uint8_t** next) {
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "array child value"));
+ const uint8_t value_metadata = *ptr++;
+ const uint8_t basic_type = value_metadata & 0x03;
+ const uint8_t value_header = value_metadata >> 2;
+ if (basic_type == 0 && value_header == 0) {
+ value->field = Field();
+ *next = ptr;
+ return Status::OK();
+ }
+ if (basic_type == 0 && value_header == 15) {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "binary value"));
+ string_values->emplace_back(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size));
+ value->field = Field::create_field<TYPE_VARBINARY>(StringView(string_values->back()));
+ value->base_scalar_type_id = TYPE_VARBINARY;
+ *next = ptr + size;
+ return Status::OK();
+ }
+
+ if (basic_type == 2 || basic_type == 3) {
+ VariantMap element_values;
+ PathInDataBuilder element_path;
+ RETURN_IF_ERROR(decode_value_to_variant_map(ptr - 1, end, metadata, &element_path,
+ &element_values, string_values, next));
+ move_variant_map_to_field(std::move(element_values), value);
+ return Status::OK();
+ }
+
+ std::string json;
Review Comment:
This fallback still makes residual VARIANT arrays fail for valid non-finite floating-point primitive elements. For an encoded array like `[NaN]` or `[Infinity]`, `decode_array_element_to_field()` does not handle the primitive float/double directly, so it reaches `decode_value()` and `append_floating_json()`. The stream output is `nan`/`inf`, which is not valid JSON, and `parse_json_to_variant_map()` fails even though the VARIANT binary encoding can represent the value. This is distinct from the existing decimal/temporal/UUID type-fidelity and binary/complex-array threads because it is a hard failure for primitive floating-point array elements caused by the remaining JSON fallback. Please decode these primitive array elements directly into `FieldWithDataType` and add coverage for residual arrays containing NaN/Infinity float or double values.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469134824
<details>
<summary>TPC-H: <b>Total hot run time: 30875 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit ea81c77fe22fe7d25590a141ce3730a04ee6d0ed, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17619 3907 3846 3846
q2 q3 10820 1379 795 795
q4 4686 475 347 347
q5 7546 2216 2125 2125
q6 251 175 137 137
q7 933 767 627 627
q8 9376 1667 1619 1619
q9 6338 4937 4867 4867
q10 6458 2068 1805 1805
q11 439 273 244 244
q12 693 422 289 289
q13 18179 3390 2744 2744
q14 266 256 235 235
q15 q16 825 789 709 709
q17 1032 945 935 935
q18 6811 5691 5516 5516
q19 1242 1232 1072 1072
q20 542 393 265 265
q21 5671 2567 2386 2386
q22 426 360 312 312
Total cold run time: 100153 ms
Total hot run time: 30875 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4266 4227 4222 4222
q2 q3 4646 5004 4425 4425
q4 2136 2252 1446 1446
q5 4493 4414 4358 4358
q6 223 174 128 128
q7 2145 1849 1644 1644
q8 2515 2094 2033 2033
q9 7923 7707 7659 7659
q10 4513 4484 4108 4108
q11 555 392 402 392
q12 821 736 522 522
q13 3307 3666 3006 3006
q14 294 298 286 286
q15 q16 775 757 665 665
q17 1338 1322 1293 1293
q18 7849 7352 6865 6865
q19 1131 1110 1120 1110
q20 2211 2221 1914 1914
q21 5277 4574 4391 4391
q22 524 490 405 405
Total cold run time: 56942 ms
Total hot run time: 50872 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469519400
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434968965
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4435969943
run p0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440944926
# BE UT Coverage Report
Increment line coverage `35.80% (653/1824)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/918c87c5454a4a3edef6fd9d50ea1e15b8d35c98_918c87c5454a4a3edef6fd9d50ea1e15b8d35c98/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/918c87c5454a4a3edef6fd9d50ea1e15b8d35c98_918c87c5454a4a3edef6fd9d50ea1e15b8d35c98/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.54% (20659/38587) |
| Line Coverage | 37.20% (195558/525737) |
| Region Coverage | 33.63% (153060/455133) |
| Branch Coverage | 34.62% (66862/193139) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441763614
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3233112122
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +118,721 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ bool has_typed_value = false;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ has_typed_value = true;
+ continue;
+ }
+ return false;
+ }
+ return has_typed_value || (has_metadata && has_value);
+}
+
+static const IColumn& remove_nullable_column(const IColumn& column) {
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(&column)) {
+ return nullable->get_nested_column();
+ }
+ return column;
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ value->precision = type->get_precision();
+ value->scale = type->get_scale();
+ return Status::OK();
+ case TYPE_FLOAT: {
+ const auto float_value = field.get<TYPE_FLOAT>();
+ value->field = std::isfinite(float_value) ? field : Field();
+ fill_variant_field_info(value);
+ return Status::OK();
+ }
+ case TYPE_DOUBLE: {
+ const auto double_value = field.get<TYPE_DOUBLE>();
+ value->field = std::isfinite(double_value) ? field : Field();
+ fill_variant_field_info(value);
+ return Status::OK();
+ }
+ case TYPE_TIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>())));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATE:
+ value->field = Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATE>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIME:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIME>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATEV2:
+ value->field =
+ Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIMEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_VARBINARY:
+ return Status::NotSupported("Parquet VARIANT binary typed_value is not supported");
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present);
+
+static Status serialize_field_to_json(const DataTypePtr& data_type, const Field& field,
+ std::string* json) {
+ MutableColumnPtr column = data_type->create_column();
+ column->insert(field);
+
+ auto json_column = ColumnString::create();
+ VectorBufferWriter writer(*json_column);
+ auto serde = data_type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ RETURN_IF_ERROR(serde->serialize_one_cell_to_json(*column, 0, writer, options));
+ writer.commit();
+ *json = json_column->get_data_at(0).to_string();
+ return Status::OK();
+}
+
+static Status scalar_typed_value_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(field_schema, field, &value, present));
+ if (!*present) {
+ return Status::OK();
+ }
+ if (value.field.is_null()) {
+ *json = "null";
+ return Status::OK();
+ }
+
+ DataTypePtr json_type;
+ if (value.base_scalar_type_id != PrimitiveType::INVALID_TYPE) {
+ json_type = DataTypeFactory::instance().create_data_type(value.base_scalar_type_id, false,
+ value.precision, value.scale);
+ } else {
+ json_type = remove_nullable(field_schema.data_type);
+ }
+ return serialize_field_to_json(json_type, value.field, json);
+}
+
+static Status variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ std::string typed_json;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_json(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, &typed_json,
+ &typed_present));
+ }
+
+ std::string value_json;
+ bool value_present = false;
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ }
+ }
+
+ if (value_present && typed_present) {
+ VariantMap value_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, PathInData(), &value_values));
+ if (value_values.find(PathInData()) != value_values.end()) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ VariantMap typed_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(typed_json, PathInData(), &typed_values));
+ value_values.merge(std::move(typed_values));
+ RETURN_IF_ERROR(variant_map_to_json(std::move(value_values), json));
+ *present = true;
+ return Status::OK();
+ }
+
+ if (typed_present) {
+ *json = std::move(typed_json);
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+
+ *present = false;
+ return Status::OK();
+}
+
+static Status shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json,
+ bool* present) {
+ if (is_variant_wrapper_field(field_schema)) {
+ return variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ return typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+static Status typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ json->clear();
+ json->push_back('[');
+ for (size_t i = 0; i < elements.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(element_schema, elements[i], metadata, &element_json,
+ &element_present));
+ if (element_present) {
+ json->append(element_json);
+ } else {
+ json->append("null");
+ }
+ }
+ json->push_back(']');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ json->clear();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(typed_value_field.children[i], fields[i], metadata,
+ &child_json, &child_present));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ first = false;
+ }
+ json->push_back('}');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ switch (typed_type->get_primitive_type()) {
+ case TYPE_STRUCT:
+ return typed_struct_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_ARRAY:
+ return typed_array_to_json(typed_value_field, field, metadata, json, present);
+ default:
+ return scalar_typed_value_to_json(typed_value_field, field, json, present);
+ }
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present);
+
+static Status variant_to_variant_map(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ VariantMap value_values;
+ bool value_present = false;
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ std::string value_json;
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, path->build(), &value_values));
+ }
+ }
+
+ VariantMap typed_values;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_variant_map(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, path,
+ &typed_values, &typed_present));
+ }
+
+ if (value_present && typed_present && value_values.find(path->build()) != value_values.end()) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ values->merge(std::move(value_values));
+ values->merge(std::move(typed_values));
+ *present = value_present || typed_present;
+ return Status::OK();
+}
+
+static Status shredded_field_to_variant_map(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (is_variant_wrapper_field(field_schema)) {
+ return variant_to_variant_map(field_schema, field, &metadata, path, values, present);
+ }
+ return typed_value_to_variant_map(field_schema, field, metadata, path, values, present);
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& fields = field.get<TYPE_STRUCT>();
+ *present = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ path->append(typed_value_field.children[i].name, false);
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_variant_map(typed_value_field.children[i], fields[i],
+ metadata, path, values, &child_present));
+ path->pop_back();
+ }
+ return Status::OK();
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ std::string value_json;
+ RETURN_IF_ERROR(
+ typed_value_to_json(typed_value_field, field, metadata, &value_json, present));
+ if (*present) {
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, path->build(), values));
+ }
+ return Status::OK();
+ }
+
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static Status append_direct_typed_column_to_batch(const FieldSchema& field_schema,
+ const IColumn& column, size_t start, size_t rows,
+ PathInDataBuilder* path, ColumnVariant* batch,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return Status::OK();
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ const auto& typed_struct = assert_cast<const ColumnStruct&>(remove_nullable_column(column));
+ return append_direct_typed_column_to_batch(field_schema.children[typed_value_idx],
+ typed_struct.get_column(typed_value_idx), start,
+ rows, path, batch, false, column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& struct_column =
Review Comment:
Fixed in 1beb1746c3e4fbef9cb87898dc837d92107b692a. The direct typed fast path now carries nullable typed_value/wrapper/struct parent null maps down to copied leaves and ORs them into the leaf null map. This keeps the columnar direct path for present rows while rows with an absent typed struct materialize as root Variant null. Added ParquetVariantReaderTest.DirectTypedOnlyPreservesNullableTypedStructNull. Local verification: ./run-be-ut.sh --run --filter="ParquetVariantReaderTest.*"; ./build.sh --be; ./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant; build-support/check-format.sh; git diff --check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4442139726
# FE UT Coverage Report
Increment line coverage ` 25.00% (1/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_2a6b8af25879042ebdd5d6314d032fd0600a22e3/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_2a6b8af25879042ebdd5d6314d032fd0600a22e3/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4433991460
Thank you for your contribution to Apache Doris.
Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR).
Please clearly describe your PR:
1. What problem was fixed (it's best to include specific error reporting information). How it was fixed.
2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be.
3. What features were added. Why was this function added?
4. Which code was refactored and why was this part of the code refactored?
5. Which functions were optimized and what is the difference before and after the optimization?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434628857
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3233846088
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +119,753 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
Review Comment:
This wrapper detection is still too broad for nested typed-only objects. A valid typed-only VARIANT can have a nested object field whose user key is named `typed_value` (for example `v.typed_value.nested.typed_value` representing `{"nested":{"typed_value":1}}`). When `can_direct_read_typed_value()` or `append_direct_typed_column_to_batch()` reaches the `nested` struct with `allow_variant_wrapper=true`, this predicate treats that ordinary nested object as a structural wrapper and unwraps the user field, so the value is inserted at path `nested` instead of `nested.typed_value`. The earlier top-level collision fix enters the top-level container with `allow_variant_wrapper=false`, but this nested case still goes through the wrapper path. Please make wrapper detection context-aware for nested typed-only structs as well, and add coverage for a nested user key named `typed_value`/`value`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3234654112
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,538 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+void append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ json->append("null");
+ return;
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, dictionary_size + 1, offset_size,
+ "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ result->dictionary.emplace_back(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
+ }
+ return Status::OK();
+}
+
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next) {
+ switch (primitive_header) {
+ case 0:
+ json->append("null");
+ *next = ptr;
+ return Status::OK();
+ case 1:
+ json->append("true");
+ *next = ptr;
+ return Status::OK();
+ case 2:
+ json->append("false");
+ *next = ptr;
+ return Status::OK();
+ case 3:
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "int8 value"));
+ json->append(std::to_string(static_cast<int8_t>(*ptr)));
+ *next = ptr + 1;
+ return Status::OK();
+ case 4:
+ RETURN_IF_ERROR(require_available(ptr, end, 2, "int16 value"));
+ json->append(std::to_string(read_signed_le(ptr, 2)));
+ *next = ptr + 2;
+ return Status::OK();
+ case 5:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "int32 value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 6:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "int64 value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 7: {
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "double value"));
+ uint64_t bits = read_unsigned_le(ptr, 8);
+ double value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 8;
+ return Status::OK();
+ }
+ case 8:
+ case 9:
+ case 10: {
+ int value_size = primitive_header == 8 ? 4 : primitive_header == 9 ? 8 : 16;
+ RETURN_IF_ERROR(require_available(ptr, end, 1 + value_size, "decimal value"));
+ int scale = static_cast<int8_t>(*ptr++);
+ __int128 unscaled = 0;
+ if (value_size == 16) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ if ((ptr[15] & 0x80) != 0) {
+ unscaled = -static_cast<__int128>((~unsigned_value) + 1);
+ } else {
+ unscaled = static_cast<__int128>(unsigned_value);
+ }
+ } else {
+ unscaled = read_signed_le(ptr, value_size);
+ }
+ append_decimal_json(unscaled, scale, json);
+ *next = ptr + value_size;
+ return Status::OK();
+ }
+ case 11:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "date value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 12:
+ case 13:
+ case 17:
+ case 18:
+ case 19:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "time or timestamp value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 14: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "float value"));
+ uint32_t bits = static_cast<uint32_t>(read_unsigned_le(ptr, 4));
+ float value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 4;
+ return Status::OK();
+ }
+ case 15: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "binary value"));
+ return Status::NotSupported("Parquet VARIANT binary primitive is not supported");
+ }
+ case 16: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary or string length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "string value"));
+ append_json_string(
+ std::string_view(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size)),
+ json);
+ *next = ptr + size;
+ return Status::OK();
+ }
+ case 20:
+ RETURN_IF_ERROR(require_available(ptr, end, 16, "uuid value"));
+ append_uuid_json(ptr, json);
+ *next = ptr + 16;
+ return Status::OK();
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT primitive header {}",
+ primitive_header);
+ }
+}
+
+Status decode_object(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, std::string* json, const uint8_t** next) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int field_id_size = ((value_header >> 2) & 0x03) + 1;
+ int num_elements_size = (value_header & 0x10) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "object element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(
+ require_available_entries(ptr, end, num_elements, field_id_size, "object field ids"));
+ std::vector<uint64_t> field_ids(num_elements);
+ for (uint64_t i = 0; i < num_elements; ++i) {
Review Comment:
`decode_object()` validates each field id is in the metadata dictionary, but it never checks the object field-id list is strictly ordered and unique. Parquet VARIANT object entries are keyed by these sorted field ids, so a malformed payload with `field_ids = [0, 0]` and two different value slices will be decoded as JSON with duplicate `"a"` keys (or later parsed with whichever duplicate-key behavior `ColumnVariant` uses) instead of returning corruption. Please reject non-increasing duplicate/out-of-order field ids here and add a unit case for duplicate field ids. This is distinct from the existing offset-bound review threads because the offsets can be valid while the logical object key list is still corrupt.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4436753148
# FE UT Coverage Report
Increment line coverage ` 25.00% (1/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_e9a298bb259994568bce23f301f467697fe37055/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_e9a298bb259994568bce23f301f467697fe37055/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3234851418
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +119,856 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return true;
Review Comment:
Fixed in the latest push by tightening wrapper detection so a struct with user fields named metadata/value is not treated as a Parquet VARIANT wrapper unless it is TYPE_VARIANT or has typed_value. Added ParquetVariantReaderTest.TypedOnlyKeepsUserMetadataAndValueFields and verified ParquetVariantReaderTest.* locally.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-5162013782
@raghav-reglobe Hi, Thanks for the feedback. We are officially working on the Iceberg variant integration, which is expected to be completed around September and October. This PR is a draft and will be closed/abandoned
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4458100633
<details>
<summary>TPC-DS: <b>Total hot run time: 169960 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit e42bb8169a7ca02afa39b1f28690edfdbe5bfa75, data reload: false
query5 4306 668 523 523
query6 346 232 205 205
query7 4350 558 315 315
query8 315 234 220 220
query9 8824 4071 4025 4025
query10 451 339 301 301
query11 5753 2382 2260 2260
query12 182 128 125 125
query13 1312 645 431 431
query14 6060 5387 5101 5101
query14_1 4464 4421 4427 4421
query15 212 212 178 178
query16 993 417 439 417
query17 1123 730 591 591
query18 2542 486 347 347
query19 220 199 159 159
query20 137 138 130 130
query21 220 140 128 128
query22 13729 13545 13376 13376
query23 17285 16361 16019 16019
query23_1 16035 16101 16087 16087
query24 7407 1779 1287 1287
query24_1 1289 1286 1297 1286
query25 537 473 409 409
query26 1305 319 169 169
query27 2747 559 342 342
query28 4449 1973 1928 1928
query29 1019 643 507 507
query30 313 242 202 202
query31 1128 1062 934 934
query32 89 85 78 78
query33 541 350 299 299
query34 1177 1147 656 656
query35 777 771 697 697
query36 1296 1325 1183 1183
query37 154 101 89 89
query38 3232 3139 3067 3067
query39 923 962 915 915
query39_1 882 872 879 872
query40 238 149 127 127
query41 67 65 63 63
query42 112 118 119 118
query43 332 346 296 296
query44
query45 211 204 194 194
query46 1124 1182 741 741
query47 2310 2296 2162 2162
query48 401 410 301 301
query49 633 498 377 377
query50 978 344 258 258
query51 4283 4250 4261 4250
query52 105 108 97 97
query53 265 287 207 207
query54 317 275 264 264
query55 91 89 83 83
query56 299 310 325 310
query57 1403 1400 1291 1291
query58 299 280 270 270
query59 1537 1594 1460 1460
query60 321 324 340 324
query61 160 154 157 154
query62 688 646 570 570
query63 247 205 204 204
query64 2421 803 628 628
query65
query66 1732 498 361 361
query67 30168 30045 29822 29822
query68
query69 468 335 302 302
query70 1045 998 927 927
query71 315 279 269 269
query72 3093 2745 2602 2602
query73 873 748 418 418
query74 5131 4916 4766 4766
query75 2704 2650 2296 2296
query76 2278 1184 761 761
query77 418 431 348 348
query78 12202 12063 11697 11697
query79 1465 1078 770 770
query80 683 578 494 494
query81 466 287 251 251
query82 1389 163 126 126
query83 364 289 261 261
query84 285 151 118 118
query85 970 631 557 557
query86 394 361 313 313
query87 3459 3361 3241 3241
query88 3636 2664 2646 2646
query89 440 395 341 341
query90 1895 188 187 187
query91 175 169 142 142
query92 77 75 77 75
query93 1457 1520 859 859
query94 550 350 317 317
query95 674 469 339 339
query96 1052 775 325 325
query97 2703 2674 2551 2551
query98 236 224 226 224
query99 1159 1105 997 997
Total cold run time: 253909 ms
Total hot run time: 169960 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467025736
<details>
<summary>TPC-DS: <b>Total hot run time: 169489 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 87943d8bbf33c9afaaa06c5545e8214cdd477a19, data reload: false
query5 4358 638 523 523
query6 343 220 200 200
query7 4218 575 289 289
query8 326 247 231 231
query9 8883 4090 4034 4034
query10 453 349 306 306
query11 5843 2381 2215 2215
query12 180 134 135 134
query13 1285 634 470 470
query14 5945 5365 5068 5068
query14_1 4378 4398 4376 4376
query15 212 210 187 187
query16 1043 460 412 412
query17 1171 752 631 631
query18 2462 503 371 371
query19 222 213 170 170
query20 140 135 133 133
query21 212 141 125 125
query22 13602 13630 13380 13380
query23 17260 16446 16117 16117
query23_1 16195 16293 16209 16209
query24 7591 1735 1297 1297
query24_1 1302 1294 1289 1289
query25 571 499 466 466
query26 1322 316 173 173
query27 2692 569 336 336
query28 4454 1970 1948 1948
query29 1042 654 518 518
query30 311 237 203 203
query31 1124 1082 952 952
query32 88 78 75 75
query33 541 350 299 299
query34 1161 1115 634 634
query35 759 787 736 736
query36 1336 1326 1167 1167
query37 153 101 90 90
query38 3212 3084 3025 3025
query39 927 939 888 888
query39_1 879 915 860 860
query40 218 149 127 127
query41 69 63 63 63
query42 111 113 108 108
query43 316 323 288 288
query44
query45 220 205 198 198
query46 1077 1160 733 733
query47 2302 2289 2159 2159
query48 381 432 307 307
query49 637 486 387 387
query50 973 341 249 249
query51 4315 4214 4245 4214
query52 105 105 93 93
query53 252 282 217 217
query54 327 280 256 256
query55 94 90 85 85
query56 303 297 298 297
query57 1401 1432 1318 1318
query58 298 268 265 265
query59 1593 1644 1428 1428
query60 320 330 306 306
query61 161 153 157 153
query62 671 636 564 564
query63 248 200 215 200
query64 2411 799 643 643
query65
query66 1718 464 357 357
query67 30015 29966 29746 29746
query68
query69 471 350 312 312
query70 1058 1010 969 969
query71 310 284 273 273
query72 3207 2748 2502 2502
query73 822 772 406 406
query74 5057 4953 4708 4708
query75 2676 2608 2252 2252
query76 2339 1130 769 769
query77 404 412 338 338
query78 12229 12048 11600 11600
query79 1448 1048 762 762
query80 1306 549 468 468
query81 534 282 234 234
query82 1172 163 123 123
query83 338 282 254 254
query84 253 141 112 112
query85 922 561 462 462
query86 447 356 307 307
query87 3480 3380 3240 3240
query88 3501 2663 2609 2609
query89 443 386 339 339
query90 1884 183 173 173
query91 181 165 143 143
query92 83 81 70 70
query93 1602 1423 855 855
query94 706 374 312 312
query95 677 478 338 338
query96 1071 784 341 341
query97 2696 2705 2608 2608
query98 241 227 228 227
query99 1090 1128 999 999
Total cold run time: 254341 ms
Total hot run time: 169489 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467669497
<details>
<summary>TPC-H: <b>Total hot run time: 30734 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit e1dce6934ce20d01059a46068509f8d202249881, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17657 3841 3881 3841
q2 q3 10787 1379 826 826
q4 4688 464 357 357
q5 7569 2223 2077 2077
q6 239 175 137 137
q7 927 787 610 610
q8 9352 1807 1573 1573
q9 6517 4907 4883 4883
q10 6428 2097 1806 1806
q11 439 273 244 244
q12 691 422 292 292
q13 18253 3373 2770 2770
q14 259 260 240 240
q15 q16 820 762 709 709
q17 978 874 1007 874
q18 6863 5627 5510 5510
q19 1273 1274 1085 1085
q20 499 384 267 267
q21 5595 2545 2333 2333
q22 428 353 300 300
Total cold run time: 100262 ms
Total hot run time: 30734 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4254 4165 4153 4153
q2 q3 4508 4873 4296 4296
q4 2089 2237 1371 1371
q5 4384 4250 4290 4250
q6 226 174 129 129
q7 2049 1868 1731 1731
q8 2402 2130 2076 2076
q9 7797 7732 7611 7611
q10 4522 4439 4035 4035
q11 579 401 382 382
q12 892 751 509 509
q13 3328 3599 3073 3073
q14 277 303 292 292
q15 q16 702 749 672 672
q17 1310 1311 1311 1311
q18 7962 7392 6903 6903
q19 1111 1085 1127 1085
q20 2196 2194 1924 1924
q21 5235 4559 4352 4352
q22 521 448 399 399
Total cold run time: 56344 ms
Total hot run time: 50554 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469193579
<details>
<summary>TPC-DS: <b>Total hot run time: 168294 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit ea81c77fe22fe7d25590a141ce3730a04ee6d0ed, data reload: false
query5 4303 663 524 524
query6 339 212 195 195
query7 4273 580 319 319
query8 335 227 217 217
query9 8865 4031 3987 3987
query10 437 323 294 294
query11 5790 2429 2238 2238
query12 184 130 125 125
query13 1268 606 417 417
query14 5968 5315 4985 4985
query14_1 4298 4289 4281 4281
query15 212 197 180 180
query16 1029 447 424 424
query17 1136 707 570 570
query18 2550 478 341 341
query19 216 201 155 155
query20 137 130 128 128
query21 213 136 114 114
query22 13558 13477 13410 13410
query23 17244 16416 15964 15964
query23_1 16204 16183 16155 16155
query24 7450 1757 1295 1295
query24_1 1300 1316 1325 1316
query25 576 511 440 440
query26 1317 316 169 169
query27 2750 545 345 345
query28 4459 1950 1975 1950
query29 1133 643 522 522
query30 304 242 205 205
query31 1114 1056 938 938
query32 91 75 75 75
query33 554 379 314 314
query34 1165 1169 664 664
query35 784 803 692 692
query36 1398 1459 1266 1266
query37 153 109 102 102
query38 3263 3189 3044 3044
query39 924 917 914 914
query39_1 881 874 879 874
query40 238 149 128 128
query41 72 69 68 68
query42 112 112 112 112
query43 328 324 287 287
query44
query45 211 203 198 198
query46 1047 1210 713 713
query47 2367 2349 2210 2210
query48 402 416 294 294
query49 646 504 397 397
query50 1017 364 245 245
query51 4324 4254 4283 4254
query52 105 107 97 97
query53 252 284 212 212
query54 325 290 262 262
query55 94 92 87 87
query56 308 314 323 314
query57 1413 1400 1302 1302
query58 314 281 278 278
query59 1508 1651 1362 1362
query60 334 340 324 324
query61 182 202 146 146
query62 664 622 558 558
query63 239 198 196 196
query64 2398 789 628 628
query65
query66 1703 459 347 347
query67 29995 29225 29738 29225
query68
query69 455 338 313 313
query70 983 978 993 978
query71 302 273 260 260
query72 2920 2263 2422 2263
query73 822 748 424 424
query74 5043 4924 4687 4687
query75 2667 2580 2254 2254
query76 2274 1142 767 767
query77 392 399 331 331
query78 12201 12164 11616 11616
query79 1473 1000 755 755
query80 1328 536 445 445
query81 502 278 238 238
query82 1181 156 123 123
query83 349 279 245 245
query84 265 141 113 113
query85 914 520 480 480
query86 444 343 318 318
query87 3412 3390 3200 3200
query88 3511 2648 2655 2648
query89 458 384 334 334
query90 1930 179 175 175
query91 177 175 138 138
query92 79 80 69 69
query93 1567 1462 921 921
query94 747 353 313 313
query95 702 467 356 356
query96 1102 775 360 360
query97 2726 2710 2583 2583
query98 237 231 221 221
query99 1125 1091 985 985
Total cold run time: 254047 ms
Total hot run time: 168294 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468992047
run cloud_p0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469603569
# FE UT Coverage Report
Increment line coverage ` 75.64% (208/275)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_87a7378a43c24b89233a6fb138ed1b79d6776812/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_87a7378a43c24b89233a6fb138ed1b79d6776812/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468964929
<details>
<summary>TPC-H: <b>Total hot run time: 30856 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 6b22fa723d69d717244b1cd0c3894901ce9a5b9b, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17672 3923 3875 3875
q2 q3 10746 1331 792 792
q4 4683 473 346 346
q5 7564 2209 2068 2068
q6 272 173 136 136
q7 993 792 617 617
q8 9364 1797 1494 1494
q9 6737 4885 4939 4885
q10 6429 2088 1782 1782
q11 433 263 243 243
q12 692 419 304 304
q13 18178 3413 2822 2822
q14 261 257 241 241
q15 q16 820 796 705 705
q17 915 879 891 879
q18 6842 5836 5529 5529
q19 1209 1183 1115 1115
q20 507 405 264 264
q21 5776 2529 2453 2453
q22 439 350 306 306
Total cold run time: 100532 ms
Total hot run time: 30856 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4212 4115 4130 4115
q2 q3 4495 4903 4345 4345
q4 2113 2205 1409 1409
q5 4419 4298 4300 4298
q6 225 174 128 128
q7 2142 1775 1693 1693
q8 2578 2100 2074 2074
q9 7917 7696 7813 7696
q10 4556 4562 4104 4104
q11 583 429 378 378
q12 859 775 510 510
q13 3395 3570 3020 3020
q14 289 295 274 274
q15 q16 725 751 639 639
q17 1330 1313 1315 1313
q18 7978 7313 7072 7072
q19 1098 1097 1136 1097
q20 2221 2225 1919 1919
q21 5281 4610 4473 4473
q22 524 469 423 423
Total cold run time: 56940 ms
Total hot run time: 50980 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468930442
<details>
<summary>TPC-H: <b>Total hot run time: 31090 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit fac97ff28873db90c5688aae3f613ac333352822, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17839 3910 3868 3868
q2 q3 10790 1394 826 826
q4 4751 478 357 357
q5 8347 2360 2133 2133
q6 377 185 139 139
q7 949 777 633 633
q8 9559 1907 1619 1619
q9 7098 4937 4937 4937
q10 6463 2101 1787 1787
q11 442 271 243 243
q12 694 420 300 300
q13 18203 3432 2765 2765
q14 258 259 233 233
q15 q16 823 771 710 710
q17 931 984 969 969
q18 6922 5837 5551 5551
q19 1187 1276 1075 1075
q20 511 408 261 261
q21 5913 2601 2388 2388
q22 446 362 296 296
Total cold run time: 102503 ms
Total hot run time: 31090 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4209 4104 4114 4104
q2 q3 4529 4978 4314 4314
q4 2103 2220 1407 1407
q5 4398 4262 4745 4262
q6 259 199 148 148
q7 1925 1858 1599 1599
q8 2468 2072 2111 2072
q9 7684 7848 7722 7722
q10 4572 4484 4267 4267
q11 589 439 367 367
q12 764 733 525 525
q13 3254 3673 2968 2968
q14 311 305 288 288
q15 q16 714 725 648 648
q17 1344 1323 1312 1312
q18 8002 7351 6702 6702
q19 1143 1126 1095 1095
q20 2227 2220 1928 1928
q21 5367 4601 4461 4461
q22 522 468 403 403
Total cold run time: 56384 ms
Total hot run time: 50592 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3254080634
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java:
##########
@@ -1058,13 +1058,23 @@ public TFileFormatType getFileFormatType() throws UserException {
if (icebergFormat.equalsIgnoreCase("parquet")) {
type = TFileFormatType.FORMAT_PARQUET;
} else if (icebergFormat.equalsIgnoreCase("orc")) {
+ validateVariantReadSupported(icebergFormat);
type = TFileFormatType.FORMAT_ORC;
} else {
throw new DdlException(String.format("Unsupported format name: %s for iceberg table.", icebergFormat));
}
return type;
}
+ private void validateVariantReadSupported(String icebergFormat) throws DdlException {
+ for (SlotDescriptor slot : desc.getSlots()) {
+ if (slot.getColumn().getType().isVariantType()) {
Review Comment:
This check only rejects slots whose top-level Doris type is `VARIANT`, but this PR maps Iceberg VARIANT recursively inside complex types. For an ORC Iceberg table with a column like `STRUCT<v: VARIANT>` (or `ARRAY<VARIANT>` / map value VARIANT), the slot type is struct/array/map, so `isVariantType()` is false and the scan reaches BE instead of failing with the Parquet-only error. Please make the read-side validation recursive over complex Doris types (or reuse the Iceberg schema traversal used for write validation) and add ORC coverage for nested VARIANT inside a complex column.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469507910
# BE UT Coverage Report
Increment line coverage `74.20% (2591/3492)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/f7d8cc0c3444ac9b3f094f645ea83ece520f3fb4_f7d8cc0c3444ac9b3f094f645ea83ece520f3fb4/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/f7d8cc0c3444ac9b3f094f645ea83ece520f3fb4_f7d8cc0c3444ac9b3f094f645ea83ece520f3fb4/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.68% (20827/38802) |
| Line Coverage | 37.38% (197512/528340) |
| Region Coverage | 33.73% (154551/458185) |
| Branch Coverage | 34.86% (67805/194523) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467119783
<details>
<summary>TPC-DS: <b>Total hot run time: 168963 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit ea6b446614717a15025ff4b0476f10c35565abc6, data reload: false
query5 4328 643 520 520
query6 352 222 204 204
query7 4231 562 289 289
query8 328 227 216 216
query9 8809 4014 3982 3982
query10 450 330 296 296
query11 5827 2396 2166 2166
query12 176 130 128 128
query13 1264 612 434 434
query14 5931 5319 5003 5003
query14_1 4317 4310 4318 4310
query15 206 203 180 180
query16 979 443 407 407
query17 934 761 586 586
query18 2451 476 353 353
query19 210 202 159 159
query20 142 138 124 124
query21 212 138 114 114
query22 13574 13556 13374 13374
query23 17142 16259 15977 15977
query23_1 16101 16140 16160 16140
query24 7414 1763 1292 1292
query24_1 1299 1297 1288 1288
query25 588 503 467 467
query26 1303 305 176 176
query27 2701 561 337 337
query28 4462 1968 1981 1968
query29 984 653 517 517
query30 316 235 204 204
query31 1117 1054 958 958
query32 87 77 79 77
query33 547 361 299 299
query34 1181 1119 625 625
query35 761 780 681 681
query36 1312 1358 1107 1107
query37 160 107 96 96
query38 3200 3154 3070 3070
query39 929 920 887 887
query39_1 885 875 863 863
query40 240 153 133 133
query41 76 69 67 67
query42 112 116 111 111
query43 322 322 295 295
query44
query45 218 207 200 200
query46 1047 1172 748 748
query47 2322 2249 2120 2120
query48 408 414 305 305
query49 635 489 385 385
query50 1012 348 250 250
query51 4327 4312 4325 4312
query52 102 109 96 96
query53 259 283 217 217
query54 326 288 271 271
query55 92 90 84 84
query56 308 315 303 303
query57 1442 1397 1348 1348
query58 306 266 269 266
query59 1567 1629 1454 1454
query60 322 326 314 314
query61 160 156 158 156
query62 662 628 551 551
query63 245 202 214 202
query64 2381 782 646 646
query65
query66 1781 466 361 361
query67 29329 29894 29755 29755
query68
query69 467 343 302 302
query70 1010 973 941 941
query71 310 275 266 266
query72 2996 2771 2475 2475
query73 851 787 432 432
query74 5059 4870 4693 4693
query75 2670 2585 2251 2251
query76 2305 1139 757 757
query77 396 415 340 340
query78 12044 12117 11617 11617
query79 1443 999 752 752
query80 648 541 451 451
query81 453 274 235 235
query82 1363 161 117 117
query83 356 276 246 246
query84 298 145 113 113
query85 872 524 449 449
query86 403 341 319 319
query87 3385 3356 3265 3265
query88 3506 2661 2629 2629
query89 432 382 336 336
query90 1934 181 180 180
query91 173 162 141 141
query92 78 77 72 72
query93 1474 1450 863 863
query94 555 358 318 318
query95 692 371 439 371
query96 980 798 323 323
query97 2700 2690 2547 2547
query98 237 228 224 224
query99 1120 1113 969 969
Total cold run time: 251433 ms
Total hot run time: 168963 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467100922
# BE UT Coverage Report
Increment line coverage `74.31% (1672/2250)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/87943d8bbf33c9afaaa06c5545e8214cdd477a19_87943d8bbf33c9afaaa06c5545e8214cdd477a19/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/87943d8bbf33c9afaaa06c5545e8214cdd477a19_87943d8bbf33c9afaaa06c5545e8214cdd477a19/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.60% (20754/38722) |
| Line Coverage | 37.31% (196779/527352) |
| Region Coverage | 33.67% (153964/457206) |
| Branch Coverage | 34.76% (67407/193900) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467449767
<details>
<summary>TPC-H: <b>Total hot run time: 30703 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit ae4500c048ebc20b3caee97cc17216107d28f7d9, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17769 3841 3840 3840
q2 q3 10744 1403 781 781
q4 4685 468 361 361
q5 7544 2207 2094 2094
q6 238 176 139 139
q7 910 790 642 642
q8 9372 1718 1674 1674
q9 5080 4905 4837 4837
q10 6380 2048 1764 1764
q11 432 271 238 238
q12 636 428 298 298
q13 18114 3369 2749 2749
q14 266 251 234 234
q15 q16 822 782 700 700
q17 953 876 918 876
q18 6883 5853 5490 5490
q19 1302 1240 1102 1102
q20 523 394 256 256
q21 5772 2496 2330 2330
q22 439 357 298 298
Total cold run time: 98864 ms
Total hot run time: 30703 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4167 4118 4086 4086
q2 q3 4479 4903 4283 4283
q4 2074 2166 1361 1361
q5 4398 4214 4313 4214
q6 229 175 130 130
q7 1728 1899 1836 1836
q8 2555 2132 2118 2118
q9 7825 7955 7608 7608
q10 4566 4494 4087 4087
q11 557 426 371 371
q12 719 727 512 512
q13 3258 3621 2982 2982
q14 290 296 275 275
q15 q16 742 763 649 649
q17 1323 1288 1274 1274
q18 7942 7114 7249 7114
q19 1157 1125 1171 1125
q20 2216 2192 1915 1915
q21 5225 4523 4413 4413
q22 501 442 395 395
Total cold run time: 55951 ms
Total hot run time: 50748 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468022540
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253710118
##########
be/src/format/parquet/parquet_nested_column_utils.cpp:
##########
@@ -0,0 +1,549 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <cctype>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "core/data_type/data_type_nullable.h"
+#include "format/parquet/schema_desc.h"
+
+namespace doris {
+namespace {
+
+enum class NestedPathMode {
+ NAME,
+ FIELD_ID,
+};
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_variant_typed_child_by_key(const FieldSchema& field_schema,
+ std::string_view key, NestedPathMode mode) {
+ if (const auto* child = find_child_by_exact_name(field_schema, key)) {
+ return child;
+ }
+ if (mode == NestedPathMode::NAME) {
+ return nullptr;
+ }
+ for (const auto& child : field_schema.children) {
+ if (child.field_id >= 0 && key == std::to_string(child.field_id)) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+bool is_unannotated_variant_value_field(const FieldSchema& field) {
+ // VARIANT residual value is raw binary; annotated strings named value are user fields.
+ return field.lower_case_name == "value" && field.physical_type == tparquet::Type::BYTE_ARRAY &&
+ !field.parquet_schema.__isset.logicalType &&
+ !field.parquet_schema.__isset.converted_type;
+}
+
+const FieldSchema* find_variant_value_field(const FieldSchema& field_schema) {
+ for (const auto& child : field_schema.children) {
+ if (is_unannotated_variant_value_field(child)) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_variant_value_field(variant_field)) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+using VariantPathMap = std::unordered_map<std::string, std::vector<std::vector<std::string>>>;
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (!is_unannotated_variant_value_field(child)) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return true;
+ }
+ if (typed_value == nullptr) {
+ return false;
+ }
+ const auto type = remove_nullable(typed_value->data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+bool add_shredded_variant_field_value(const FieldSchema& shredded_field,
+ std::set<uint64_t>& column_ids) {
+ if (const auto* value = find_variant_value_field(shredded_field)) {
+ add_column_id_range(*value, column_ids);
+ return true;
+ }
+ return false;
+}
+
+bool is_variant_array_subscript(std::string_view path) {
+ return !path.empty() &&
+ std::all_of(path.begin(), path.end(), [](unsigned char c) { return std::isdigit(c); });
+}
+
+bool is_terminal_variant_meta_component(std::string_view path) {
+ return path == "NULL" || path == "OFFSET";
+}
+
+const std::vector<std::string>& effective_variant_path(const std::vector<std::string>& raw_path,
+ std::vector<std::string>& stripped_path) {
+ if (!raw_path.empty() && is_terminal_variant_meta_component(raw_path.back())) {
+ stripped_path.assign(raw_path.begin(), raw_path.end() - 1);
+ return stripped_path;
+ }
+ return raw_path;
+}
+
+bool contains_inherited_metadata_value(const FieldSchema& field_schema) {
+ if (is_shredded_variant_field(field_schema) &&
+ find_variant_value_field(field_schema) != nullptr) {
+ return true;
+ }
+ return std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_inherited_metadata_value(child); });
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode);
+
+VariantColumnIdExtractionResult extract_typed_value_path(const FieldSchema& typed_value,
+ const std::vector<std::string>& path,
+ std::set<uint64_t>& column_ids,
+ NestedPathMode mode) {
+ VariantColumnIdExtractionResult result;
+ const auto typed_value_type = remove_nullable(typed_value.data_type);
+ if (typed_value_type->get_primitive_type() != TYPE_STRUCT) {
+ result = extract_variant_typed_nested_column_ids(typed_value, {path}, column_ids, mode);
+ } else if (const auto* typed_child =
+ find_variant_typed_child_by_key(typed_value, path[0], mode)) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata = contains_inherited_metadata_value(*typed_child);
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ result = extract_variant_typed_nested_column_ids(*typed_child, child_paths, column_ids,
+ mode);
+ }
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(typed_value.get_column_id());
+ }
+ return result;
+}
+
+void add_variant_typed_path(PrimitiveType field_type, const FieldSchema& field_schema,
+ const std::vector<std::string>& path,
+ VariantColumnIdExtractionResult* result, std::set<uint64_t>& column_ids,
+ VariantPathMap* child_paths) {
+ if (path.empty()) {
+ add_column_id_range(field_schema, column_ids);
+ result->has_child_columns = true;
+ result->needs_metadata |= contains_inherited_metadata_value(field_schema);
+ return;
+ }
+
+ const bool is_list = field_type == PrimitiveType::TYPE_ARRAY;
+ const bool is_map = field_type == PrimitiveType::TYPE_MAP;
+ std::vector<std::string> remaining;
+ std::string child_key;
+ if (is_list) {
+ child_key = "*";
+ if (!is_variant_array_subscript(path[0])) {
+ remaining.assign(path.begin(), path.end());
+ } else if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ } else if (is_map) {
+ (*child_paths)["KEYS"].emplace_back();
+ child_key = "VALUES";
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ } else {
+ child_key = path[0];
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ }
+ (*child_paths)[child_key].push_back(std::move(remaining));
+}
+
+std::string variant_typed_child_key(PrimitiveType field_type, const FieldSchema& field_schema,
+ uint64_t child_index) {
+ if (field_type == PrimitiveType::TYPE_ARRAY) {
+ return "*";
+ }
+ if (field_type == PrimitiveType::TYPE_MAP) {
+ if (child_index == 0) {
+ return "KEYS";
+ }
+ return child_index == 1 ? "VALUES" : "";
+ }
+ return field_schema.children[child_index].name;
+}
+
+void append_variant_child_paths(const VariantPathMap& paths_by_name, const std::string& key,
+ std::vector<std::vector<std::string>>& child_paths) {
+ auto child_paths_it = paths_by_name.find(key);
+ if (child_paths_it != paths_by_name.end()) {
+ child_paths.insert(child_paths.end(), child_paths_it->second.begin(),
+ child_paths_it->second.end());
+ }
+}
+
+std::vector<std::vector<std::string>> collect_variant_typed_child_paths(
+ const VariantPathMap& paths_by_name, const FieldSchema& child, const std::string& child_key,
+ bool is_list, bool is_map, NestedPathMode mode) {
+ std::vector<std::vector<std::string>> child_paths;
+ append_variant_child_paths(paths_by_name, child_key, child_paths);
+ if (!is_list && !is_map && mode == NestedPathMode::FIELD_ID && child.field_id >= 0) {
+ const std::string field_id_key = std::to_string(child.field_id);
+ if (field_id_key != child_key) {
+ append_variant_child_paths(paths_by_name, field_id_key, child_paths);
+ }
+ }
+ return child_paths;
+}
+
+void extract_variant_typed_child_column_ids(
+ const FieldSchema& child, const std::vector<std::vector<std::string>>& child_paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode,
+ VariantColumnIdExtractionResult* result) {
+ const bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+ if (needs_full_child) {
+ add_column_id_range(child, column_ids);
+ result->has_child_columns = true;
+ result->needs_metadata |= contains_inherited_metadata_value(child);
+ return;
+ }
+
+ auto child_result =
+ extract_variant_typed_nested_column_ids(child, child_paths, column_ids, mode);
+ result->has_child_columns |= child_result.has_child_columns;
+ result->needs_metadata |= child_result.needs_metadata;
+}
+
+VariantColumnIdExtractionResult extract_shredded_variant_field_ids(
+ const FieldSchema& shredded_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode) {
+ const auto* typed_value = find_child_by_structural_name(shredded_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& raw_path : paths) {
+ std::vector<std::string> stripped_path;
+ const auto& path = effective_variant_path(raw_path, stripped_path);
+ if (path.empty()) {
+ add_column_id_range(shredded_field, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata |= contains_inherited_metadata_value(shredded_field);
+ continue;
+ }
+
+ VariantColumnIdExtractionResult typed_result;
+ if (typed_value != nullptr) {
+ typed_result = extract_typed_value_path(*typed_value, path, column_ids, mode);
+ result.needs_metadata |= typed_result.needs_metadata;
+ }
+ const bool has_residual_value =
+ add_shredded_variant_field_value(shredded_field, column_ids);
+ if (has_residual_value) {
+ result.needs_metadata = true;
+ }
+ if (!typed_result.has_child_columns) {
+ result.has_child_columns |= has_residual_value;
+ continue;
+ }
+ result.has_child_columns = true;
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(shredded_field.get_column_id());
+ }
+ return result;
+}
+
+VariantColumnIdExtractionResult extract_variant_nested_column_ids(
+ const FieldSchema& variant_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode) {
+ const auto* typed_value = find_child_by_structural_name(variant_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& raw_path : paths) {
+ std::vector<std::string> stripped_path;
+ const auto& path = effective_variant_path(raw_path, stripped_path);
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ result.has_child_columns = true;
+ continue;
+ }
+
+ VariantColumnIdExtractionResult typed_result;
+ if (typed_value != nullptr) {
+ typed_result = extract_typed_value_path(*typed_value, path, column_ids, mode);
+ if (typed_result.needs_metadata) {
+ add_variant_metadata(variant_field, column_ids);
+ }
+ }
+
+ if (!typed_result.has_child_columns) {
+ add_variant_value(variant_field, column_ids);
+ }
+ result.has_child_columns = true;
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(variant_field.get_column_id());
+ }
+ return result;
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode) {
+ if (remove_nullable(field_schema.data_type)->get_primitive_type() ==
+ PrimitiveType::TYPE_VARIANT) {
+ return extract_variant_nested_column_ids(field_schema, paths, column_ids, mode);
+ }
+ if (is_shredded_variant_field(field_schema)) {
+ return extract_shredded_variant_field_ids(field_schema, paths, column_ids, mode);
+ }
+
+ VariantColumnIdExtractionResult result;
+ VariantPathMap child_paths_by_name;
+ const auto field_type = remove_nullable(field_schema.data_type)->get_primitive_type();
+ for (const auto& path : paths) {
+ add_variant_typed_path(field_type, field_schema, path, &result, column_ids,
+ &child_paths_by_name);
+ }
+
+ const bool is_list = field_type == PrimitiveType::TYPE_ARRAY;
+ const bool is_map = field_type == PrimitiveType::TYPE_MAP;
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+ const std::string child_key = variant_typed_child_key(field_type, field_schema, i);
+ auto child_paths = collect_variant_typed_child_paths(child_paths_by_name, child, child_key,
+ is_list, is_map, mode);
+ if (child_paths.empty()) {
+ continue;
+ }
+ extract_variant_typed_child_column_ids(child, child_paths, column_ids, mode, &result);
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(field_schema.get_column_id());
+ }
+ return result;
+}
+
+void normalize_map_wildcard(
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>>& child_paths) {
+ auto wildcard_it = child_paths.find("*");
+ if (wildcard_it == child_paths.end()) {
+ return;
+ }
+
+ auto wildcard_paths = std::move(wildcard_it->second);
+ child_paths.erase(wildcard_it);
+ auto& values_paths = child_paths["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+ child_paths["KEYS"].emplace_back();
+}
+
+std::string get_nested_child_key(const FieldSchema& field_schema, uint64_t child_index,
+ NestedPathMode mode) {
+ const auto field_type = remove_nullable(field_schema.data_type)->get_primitive_type();
+ if (field_type == PrimitiveType::TYPE_ARRAY) {
+ return "*";
+ }
+ if (field_type == PrimitiveType::TYPE_MAP) {
+ if (child_index == 0) {
+ return "KEYS";
+ }
+ return child_index == 1 ? "VALUES" : "";
+ }
+
+ const auto& child = field_schema.children[child_index];
+ if (mode == NestedPathMode::NAME) {
+ return child.lower_case_name;
+ }
+ return std::to_string(child.field_id);
+}
+
+bool should_skip_nested_child_key(std::string_view child_key, NestedPathMode mode) {
+ return child_key.empty() || (mode == NestedPathMode::FIELD_ID && child_key == "-1");
+}
+
+void extract_nested_column_ids_impl(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids, NestedPathMode mode) {
+ const auto field_type = remove_nullable(field_schema.data_type)->get_primitive_type();
+ if (field_type == PrimitiveType::TYPE_VARIANT) {
+ static_cast<void>(extract_variant_nested_column_ids(field_schema, paths, column_ids, mode));
+ return;
+ }
+
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>> child_paths_by_key;
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ continue;
+ }
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_key[path[0]].push_back(std::move(remaining));
+ }
+
+ if (field_type == PrimitiveType::TYPE_MAP) {
Review Comment:
The new `cardinality(arr)` access collection emits `[arr, OFFSET]`, but this extractor only special-cases OFFSET for MAP below. For an ARRAY slot, `child_paths_by_key` contains only `OFFSET` while `get_nested_child_key()` returns `*`, so no element child id is selected. `ParquetColumnReader::create()` for ARRAY then still recurses into the element reader unconditionally and `ArrayColumnReader::read_column_data()` decodes the element column just to obtain rep/def levels, so `SELECT cardinality(arr) FROM parquet_array_tbl` does not get the offset-only behavior this PR adds and can still read large element payloads. Please either normalize ARRAY OFFSET to a real levels-only/element reader path or add a dedicated array-offset reader, and cover root/nested ARRAY cardinality on standalone/Hive/Iceberg Parquet. This is distinct from the existing MAP OFFSET thread because ARRAY does not crash, but it silently defeats the intended pruning optimization.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468480553
# FE UT Coverage Report
Increment line coverage ` 73.24% (104/142)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_35f02662f9c320859ede78361f2d1e6474a1261a/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_35f02662f9c320859ede78361f2d1e6474a1261a/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468495231
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468747536
<details>
<summary>TPC-DS: <b>Total hot run time: 171274 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 0b5f089ab1c302d77f7a29632bfb66db8b919dcd, data reload: false
query5 4321 666 536 536
query6 366 227 203 203
query7 4273 566 315 315
query8 334 234 230 230
query9 8849 4037 4011 4011
query10 450 338 310 310
query11 5801 2361 2295 2295
query12 193 133 125 125
query13 1303 632 429 429
query14 6000 5383 5083 5083
query14_1 4421 4362 4381 4362
query15 220 209 181 181
query16 1023 466 449 449
query17 1160 742 606 606
query18 2578 515 362 362
query19 238 212 173 173
query20 145 134 135 134
query21 227 143 119 119
query22 13685 13782 13632 13632
query23 17580 16733 16072 16072
query23_1 16167 16190 16256 16190
query24 7429 1787 1310 1310
query24_1 1301 1294 1309 1294
query25 591 501 447 447
query26 1309 334 178 178
query27 2765 551 358 358
query28 4437 1954 1943 1943
query29 1053 641 521 521
query30 318 254 203 203
query31 1132 1084 938 938
query32 95 78 76 76
query33 548 364 312 312
query34 1221 1161 640 640
query35 790 775 731 731
query36 1322 1316 1157 1157
query37 153 105 87 87
query38 3199 3161 3060 3060
query39 943 921 903 903
query39_1 867 914 909 909
query40 229 142 125 125
query41 65 64 63 63
query42 111 113 110 110
query43 318 331 294 294
query44
query45 207 197 193 193
query46 1073 1201 739 739
query47 2306 2361 2264 2264
query48 425 429 291 291
query49 658 497 383 383
query50 975 340 258 258
query51 4327 4316 4261 4261
query52 111 111 100 100
query53 264 297 211 211
query54 332 278 258 258
query55 99 93 85 85
query56 313 319 317 317
query57 1485 1419 1333 1333
query58 299 265 272 265
query59 1591 1617 1449 1449
query60 313 327 315 315
query61 159 155 149 149
query62 674 622 563 563
query63 241 208 211 208
query64 2327 810 647 647
query65
query66 1717 480 356 356
query67 30215 29627 29932 29627
query68
query69 481 341 327 327
query70 936 1019 1005 1005
query71 308 290 276 276
query72 3028 2689 2416 2416
query73 817 778 420 420
query74 5081 4940 4736 4736
query75 2693 2601 2259 2259
query76 2301 1134 758 758
query77 409 421 338 338
query78 12183 12221 11697 11697
query79 1507 1042 710 710
query80 1308 548 446 446
query81 525 282 236 236
query82 992 154 121 121
query83 322 272 246 246
query84 257 139 110 110
query85 934 523 446 446
query86 462 356 310 310
query87 3451 3343 3211 3211
query88 3542 2671 2652 2652
query89 453 380 337 337
query90 1912 188 181 181
query91 177 171 141 141
query92 79 80 74 74
query93 1553 1446 910 910
query94 752 361 291 291
query95 694 463 348 348
query96 999 811 363 363
query97 2695 2735 2597 2597
query98 253 236 236 236
query99 1089 1112 987 987
Total cold run time: 254901 ms
Total hot run time: 171274 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468150097
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467049271
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253166929
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java:
##########
@@ -254,6 +259,9 @@ public Void visitLogicalProject(LogicalProject<? extends Plan> project, Statemen
)
);
}
+ } else if (collectWholeVariantOutput && output instanceof Alias && output.child(0) instanceof Slot
+ && collectWholeVariantOutput((Slot) output.child(0))) {
+ continue;
} else {
exprCollector.collect(output);
}
Review Comment:
Collecting project expressions here still misses non-literal VARIANT subscripts, which are unprunable and must force a full root read. For example, `SELECT v[cast(id AS string)] FROM t WHERE v['k'] IS NOT NULL` reaches `visitElementAt()` with a VARIANT first argument but a non-literal key, falls through to the default visitor, and visits `SlotReference v` with an empty builder; `visitSlotReference()` records no root `[v]` path. The filter can still record only `[v, k]`, so the scan may prune siblings even though the dynamic lookup can access any key. This is distinct from the existing full-value-expression thread because the full-column requirement comes from a dynamic `ElementAt`, not `cast(v)` or a bare slot projection. Please make non-literal VARIANT subscripts record a root access path and add coverage with a nested predicate present at the same scan.
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,951 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <deque>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+#include "core/column/column_variant.h"
+#include "core/value/jsonb_value.h"
+#include "exec/common/variant_util.h"
+
+namespace doris::parquet {
+
+std::string format_variant_uuid(const uint8_t* ptr) {
+ static constexpr char hex[] = "0123456789abcdef";
+ std::string uuid;
+ uuid.reserve(36);
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ uuid.push_back('-');
+ }
+ uuid.push_back(hex[ptr[i] >> 4]);
+ uuid.push_back(hex[ptr[i] & 0x0f]);
+ }
+ return uuid;
+}
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+struct VariantObjectLayout {
+ std::vector<uint64_t> field_ids;
+ std::vector<uint64_t> field_offsets;
+ std::vector<uint64_t> field_ends;
+ const uint8_t* fields = nullptr;
+ uint64_t total_size = 0;
+};
+
+struct VariantArrayLayout {
+ std::vector<uint64_t> field_offsets;
+ const uint8_t* fields = nullptr;
+ uint64_t total_size = 0;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+__int128 read_signed_int128_le(const uint8_t* ptr) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ static constexpr unsigned __int128 sign_bit = static_cast<unsigned __int128>(1) << 127;
+ if ((unsigned_value & sign_bit) == 0) {
+ return static_cast<__int128>(unsigned_value);
+ }
+ static constexpr __int128 signed_half_range = static_cast<__int128>(1) << 126;
+ return (static_cast<__int128>(unsigned_value & (sign_bit - 1)) - signed_half_range) -
+ signed_half_range;
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ if (size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+bool variant_string_less(std::string_view lhs, std::string_view rhs) {
+ return std::lexicographical_compare(
+ lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](char left, char right) {
+ return static_cast<unsigned char>(left) < static_cast<unsigned char>(right);
+ });
+}
+
+bool is_valid_utf8(std::string_view value) {
+ const auto* data = reinterpret_cast<const uint8_t*>(value.data());
+ const auto* end = data + value.size();
+ while (data < end) {
+ const uint8_t first = *data++;
+ if (first <= 0x7f) {
+ continue;
+ }
+
+ uint32_t code_point = 0;
+ size_t continuation_bytes = 0;
+ if (first >= 0xc2 && first <= 0xdf) {
+ code_point = first & 0x1f;
+ continuation_bytes = 1;
+ } else if (first >= 0xe0 && first <= 0xef) {
+ code_point = first & 0x0f;
+ continuation_bytes = 2;
+ } else if (first >= 0xf0 && first <= 0xf4) {
+ code_point = first & 0x07;
+ continuation_bytes = 3;
+ } else {
+ return false;
+ }
+
+ if (static_cast<size_t>(end - data) < continuation_bytes) {
+ return false;
+ }
+ for (size_t i = 0; i < continuation_bytes; ++i) {
+ const uint8_t byte = *data++;
+ if ((byte & 0xc0) != 0x80) {
+ return false;
+ }
+ code_point = (code_point << 6) | (byte & 0x3f);
+ }
+
+ if ((continuation_bytes == 2 && code_point < 0x800) ||
+ (continuation_bytes == 3 && code_point < 0x10000) ||
+ (code_point >= 0xd800 && code_point <= 0xdfff) || code_point > 0x10ffff) {
+ return false;
+ }
+ }
+ return true;
+}
+
+Status require_valid_utf8(std::string_view value, std::string_view context) {
+ if (!is_valid_utf8(value)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} UTF-8 string", context);
+ }
+ return Status::OK();
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+Status append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point value is not supported");
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+ return Status::OK();
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next);
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ json->push_back('"');
+ json->append(format_variant_uuid(ptr));
+ json->push_back('"');
+}
+
+Status make_jsonb_field(std::string_view json, FieldWithDataType* value) {
+ JsonBinaryValue jsonb_value;
+ RETURN_IF_ERROR(jsonb_value.from_json_string(json.data(), json.size()));
+ value->field =
+ Field::create_field<TYPE_JSONB>(JsonbField(jsonb_value.value(), jsonb_value.size()));
+ value->base_scalar_type_id = TYPE_JSONB;
+ return Status::OK();
+}
+
+Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field("{}", &value));
+ (*values)[path] = std::move(value);
+ return Status::OK();
+}
+
+Status parse_json_to_variant_map(std::string_view json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (parsed.is_null()) {
+ (*values)[prefix] = FieldWithDataType {.field = Field()};
+ return Status::OK();
+ }
+
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ for (auto& [parsed_path, value] : parsed.get<TYPE_VARIANT>()) {
+ path.append(parsed_path.get_parts(), false);
+ (*values)[path.build()] = std::move(value);
+ for (size_t i = 0; i < parsed_path.get_parts().size(); ++i) {
+ path.pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+void fill_field_type_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+ value->precision = info.precision;
+ value->scale = info.scale;
+}
+
+Status read_array_layout(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ VariantArrayLayout* layout) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int num_elements_size = (value_header & 0x04) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "array element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "array field offsets"));
+ layout->field_offsets.resize(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ layout->field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ layout->total_size = layout->field_offsets.back();
+ layout->fields = ptr;
+ RETURN_IF_ERROR(
+ require_available(layout->fields, end, layout->total_size, "array field values"));
+ RETURN_IF_ERROR(
+ validate_array_field_offsets(layout->field_offsets, layout->total_size, "array"));
+ return Status::OK();
+}
+
+Status read_object_layout(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, VariantObjectLayout* layout) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int field_id_size = ((value_header >> 2) & 0x03) + 1;
+ int num_elements_size = (value_header & 0x10) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "object element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(
+ require_available_entries(ptr, end, num_elements, field_id_size, "object field ids"));
+ layout->field_ids.resize(num_elements);
+ for (uint64_t i = 0; i < num_elements; ++i) {
+ layout->field_ids[i] = read_unsigned_le(ptr, field_id_size);
+ ptr += field_id_size;
+ if (layout->field_ids[i] >= metadata.dictionary.size()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field id {}",
+ layout->field_ids[i]);
+ }
+ if (i > 0 && !variant_string_less(metadata.dictionary[layout->field_ids[i - 1]],
+ metadata.dictionary[layout->field_ids[i]])) {
+ return Status::Corruption("Invalid Parquet VARIANT object field names");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "object field offsets"));
+ layout->field_offsets.resize(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ layout->field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ layout->total_size = layout->field_offsets.back();
+ layout->fields = ptr;
+ RETURN_IF_ERROR(
+ require_available(layout->fields, end, layout->total_size, "object field values"));
+ RETURN_IF_ERROR(compute_object_field_ends(layout->field_offsets, layout->total_size,
+ &layout->field_ends));
+ return Status::OK();
+}
+
+Status decode_value_to_variant_map(const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, PathInDataBuilder* path,
+ VariantMap* values, std::deque<std::string>* string_values,
+ const uint8_t** next);
+
+Status decode_primitive_to_variant_map(uint8_t primitive_header, const uint8_t* ptr,
+ const uint8_t* end, const VariantMetadata&,
+ PathInDataBuilder* path, VariantMap* values,
+ std::deque<std::string>* string_values,
+ const uint8_t** next) {
+ if (primitive_header == 15) {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "binary value"));
+ string_values->emplace_back(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size));
+ (*values)[path->build()] = FieldWithDataType {
+ .field = Field::create_field<TYPE_VARBINARY>(StringView(string_values->back())),
+ .base_scalar_type_id = TYPE_VARBINARY};
+ *next = ptr + size;
+ return Status::OK();
+ }
+
+ std::string json;
+ RETURN_IF_ERROR(decode_primitive(primitive_header, ptr, end, &json, next));
+ return parse_json_to_variant_map(json, path->build(), values);
+}
+
+Status decode_object_to_variant_map(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, PathInDataBuilder* path,
+ VariantMap* values, std::deque<std::string>* string_values,
+ const uint8_t** next) {
+ VariantObjectLayout layout;
+ RETURN_IF_ERROR(read_object_layout(value_header, ptr, end, metadata, &layout));
+
+ if (layout.field_ids.empty()) {
+ RETURN_IF_ERROR(insert_empty_object_marker(path->build(), values));
+ }
+
+ for (uint64_t i = 0; i < layout.field_ids.size(); ++i) {
+ const uint8_t* child_begin = layout.fields + layout.field_offsets[i];
+ const uint8_t* child_end = layout.fields + layout.field_ends[i];
+ const uint8_t* child_next = nullptr;
+ path->append(metadata.dictionary[layout.field_ids[i]], false);
+ RETURN_IF_ERROR(decode_value_to_variant_map(child_begin, child_end, metadata, path, values,
+ string_values, &child_next));
+ path->pop_back();
+ if (child_next != child_end) {
+ return Status::Corruption("Invalid Parquet VARIANT object child value length");
+ }
+ }
+ *next = layout.fields + layout.total_size;
+ return Status::OK();
+}
+
+Status decode_scalar_array_element_to_field(const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata,
+ FieldWithDataType* value,
+ std::deque<std::string>* string_values,
+ const uint8_t** next) {
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "array child value"));
+ const uint8_t value_metadata = *ptr++;
+ const uint8_t basic_type = value_metadata & 0x03;
+ const uint8_t value_header = value_metadata >> 2;
+ if (basic_type == 0 && value_header == 0) {
+ value->field = Field();
+ *next = ptr;
+ return Status::OK();
+ }
+ if (basic_type == 0 && value_header == 15) {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "binary length"));
+ uint64_t size = read_unsigned_le(ptr, 4);
+ ptr += 4;
+ RETURN_IF_ERROR(require_available(ptr, end, size, "binary value"));
+ string_values->emplace_back(reinterpret_cast<const char*>(ptr), static_cast<size_t>(size));
+ value->field = Field::create_field<TYPE_VARBINARY>(StringView(string_values->back()));
+ value->base_scalar_type_id = TYPE_VARBINARY;
+ *next = ptr + size;
+ return Status::OK();
+ }
+ if (basic_type != 0 && basic_type != 1) {
+ return Status::NotSupported("Parquet VARIANT residual array contains complex element");
Review Comment:
This scalar-only residual array path still rejects valid arrays whose elements are complex values containing binary primitives. For example, a residual value like `[{"b": binary(0xff)}]` enters `decode_array_to_variant_map()`, reaches this branch for the object element, returns `NotSupported`, then falls back to JSON decoding; that fallback still fails when `decode_primitive()` sees the binary primitive inside the object. The same binary object works outside an array via `decode_object_to_variant_map()`, so this is a distinct remaining array-element case from the existing `[binary(...)]` thread. Please decode complex array elements into `VariantMap` without the JSON fallback (or otherwise make binary-aware complex array reconstruction work) and add coverage for an array containing an object with a binary field.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +127,1480 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static bool is_value_only_variant_wrapper_candidate(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_value = false;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "value" && child.physical_type == tparquet::Type::BYTE_ARRAY) {
+ has_value = true;
+ continue;
+ }
+ return false;
+ }
+ return has_value;
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status make_jsonb_field(std::string_view json, FieldWithDataType* value) {
+ JsonBinaryValue jsonb_value;
+ RETURN_IF_ERROR(jsonb_value.from_json_string(json.data(), json.size()));
+ value->field =
+ Field::create_field<TYPE_JSONB>(JsonbField(jsonb_value.value(), jsonb_value.size()));
+ value->base_scalar_type_id = TYPE_JSONB;
+ return Status::OK();
+}
+
+static Status make_empty_object_field(Field* field) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field("{}", &value));
+ *field = std::move(value.field);
+ return Status::OK();
+}
+
+static Status insert_jsonb_value(const PathInData& path, std::string_view json,
+ VariantMap* values) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field(json, &value));
+ (*values)[path] = std::move(value);
+ return Status::OK();
+}
+
+static Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ return insert_jsonb_value(path, "{}", values);
+}
+
+static bool is_empty_object_marker(const FieldWithDataType& value) {
+ if (value.field.get_type() != TYPE_JSONB) {
+ return false;
+ }
+ const auto& jsonb = value.field.get<TYPE_JSONB>();
+ const JsonbDocument* document = nullptr;
+ Status st =
+ JsonbDocument::checkAndCreateDocument(jsonb.get_value(), jsonb.get_size(), &document);
+ if (!st.ok() || document == nullptr || document->getValue() == nullptr ||
+ !document->getValue()->isObject()) {
+ return false;
+ }
+ return document->getValue()->unpack<ObjectVal>()->numElem() == 0;
+}
+
+static Status collect_empty_object_markers(const rapidjson::Value& value, PathInDataBuilder* path,
+ VariantMap* values) {
+ if (!value.IsObject()) {
+ return Status::OK();
+ }
+ if (value.MemberCount() == 0) {
+ return insert_empty_object_marker(path->build(), values);
+ }
+ for (auto it = value.MemberBegin(); it != value.MemberEnd(); ++it) {
+ if (it->value.IsObject()) {
+ path->append(std::string_view(it->name.GetString(), it->name.GetStringLength()), false);
+ RETURN_IF_ERROR(collect_empty_object_markers(it->value, path, values));
+ path->pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+static Status add_empty_object_markers_from_json(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ if (json.find("{}") == std::string::npos) {
+ return Status::OK();
+ }
+ rapidjson::Document document;
+ document.Parse(json.data(), json.size());
+ if (document.HasParseError()) {
+ return Status::Corruption("Invalid Parquet VARIANT decoded JSON");
+ }
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ return collect_empty_object_markers(document, &path, values);
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (!parsed.is_null()) {
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ }
+ RETURN_IF_ERROR(add_empty_object_markers_from_json(json, prefix, values));
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static bool path_has_prefix(const PathInData& path, const PathInData& prefix) {
+ const auto& parts = path.get_parts();
+ const auto& prefix_parts = prefix.get_parts();
+ if (parts.size() < prefix_parts.size()) {
+ return false;
+ }
+ for (size_t i = 0; i < prefix_parts.size(); ++i) {
+ if (parts[i] != prefix_parts[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool has_descendant_path(const VariantMap& values, const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ return std::ranges::any_of(values, [&](const auto& entry) {
+ const auto& path = entry.first;
+ return path.get_parts().size() > prefix_size && path_has_prefix(path, prefix);
+ });
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* values,
+ const VariantMap& shadowing_values) {
+ for (auto it = values->begin(); it != values->end();) {
+ if (is_empty_object_marker(it->second) &&
+ (has_descendant_path(*values, it->first) ||
+ has_descendant_path(shadowing_values, it->first))) {
+ it = values->erase(it);
+ continue;
+ }
+ ++it;
+ }
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* value_values,
+ VariantMap* typed_values) {
+ erase_shadowed_empty_object_markers(value_values, *typed_values);
+ erase_shadowed_empty_object_markers(typed_values, *value_values);
+}
+
+static Status check_no_shredded_value_typed_duplicates(const VariantMap& value_values,
+ const VariantMap& typed_values,
+ const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ for (const auto& value_entry : value_values) {
+ const auto& value_path = value_entry.first;
+ if (!path_has_prefix(value_path, prefix)) {
+ continue;
+ }
+ if (value_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(value_entry.second) &&
+ !has_descendant_path(typed_values, value_path)) {
+ continue;
+ }
+ if (!typed_values.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT residual value conflicts with typed_value at path {}",
+ value_path.get_path());
+ }
+ continue;
+ }
+ for (const auto& typed_entry : typed_values) {
+ const auto& typed_path = typed_entry.first;
+ if (!path_has_prefix(typed_path, prefix)) {
+ continue;
+ }
+ if (typed_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(typed_entry.second) &&
+ !has_descendant_path(value_values, typed_path)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ if (value_path.get_parts()[prefix_size] == typed_path.get_parts()[prefix_size]) {
+ if (value_path == typed_path && is_empty_object_marker(value_entry.second) &&
+ is_empty_object_marker(typed_entry.second)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+static bool has_direct_typed_parent_null(const std::vector<const NullMap*>& null_maps, size_t row) {
+ return std::ranges::any_of(null_maps, [&](const NullMap* null_map) {
+ DCHECK_LT(row, null_map->size());
+ return (*null_map)[row];
+ });
+}
+
+static void insert_direct_typed_leaf_range(const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps,
+ IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ nullable_leaf.get_nested_column().insert_range_from(*value_column, start, rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static bool is_temporal_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_uuid_typed_value_field(const FieldSchema& field_schema);
+static bool contains_uuid_typed_value_field(const FieldSchema& field_schema);
+
+static DataTypePtr direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return std::make_shared<DataTypeInt64>();
+ }
+ return type;
+}
+
+static DataTypePtr direct_variant_leaf_type(const FieldSchema& field_schema) {
+ if (is_uuid_typed_value_field(field_schema)) {
+ return std::make_shared<DataTypeString>();
+ }
+ return direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool contains_temporal_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_temporal_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static int64_t direct_temporal_variant_value(PrimitiveType type, const IColumn& column,
+ size_t row) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ return static_cast<int64_t>(
+ std::llround(assert_cast<const ColumnTimeV2&>(column).get_data()[row]));
+ case TYPE_DATE:
+ return variant_date_value(assert_cast<const ColumnDate&>(column).get_data()[row]);
+ case TYPE_DATETIME:
+ return variant_datetime_value(assert_cast<const ColumnDateTime&>(column).get_data()[row]);
+ case TYPE_DATEV2:
+ return variant_date_value(assert_cast<const ColumnDateV2&>(column).get_data()[row]);
+ case TYPE_DATETIMEV2:
+ return variant_datetime_value(assert_cast<const ColumnDateTimeV2&>(column).get_data()[row]);
+ case TYPE_TIMESTAMPTZ:
+ return variant_datetime_value(
+ assert_cast<const ColumnTimeStampTz&>(column).get_data()[row]);
+ default:
+ DORIS_CHECK(false);
+ return 0;
+ }
+}
+
+static void insert_direct_typed_temporal_leaf_range(
+ PrimitiveType type, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnInt64&>(nullable_leaf.get_nested_column()).get_data();
+ data.reserve(data.size() + rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ data.push_back(direct_temporal_variant_value(type, *value_column, row));
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static Status insert_direct_typed_uuid_leaf_range(
+ const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnString&>(nullable_leaf.get_nested_column());
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ const bool is_null = leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row);
+ if (is_null) {
+ data.insert_default();
+ null_map.push_back(1);
+ continue;
+ }
+ StringRef bytes = value_column->get_data_at(row);
+ if (bytes.size != 16) {
+ return Status::Corruption("Parquet VARIANT UUID typed_value has invalid length {}",
+ bytes.size);
+ }
+ std::string uuid =
+ parquet::format_variant_uuid(reinterpret_cast<const uint8_t*>(bytes.data));
+ data.insert_data(uuid.data(), uuid.size());
+ null_map.push_back(0);
+ }
+ return Status::OK();
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY:
+ return true;
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return !contains_temporal_variant_leaf_type(array_type->get_nested_type()) &&
+ is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY && contains_uuid_typed_value_field(field_schema)) {
+ return false;
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool has_selected_direct_typed_leaf(const FieldSchema& field_schema,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return false;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ return has_selected_direct_typed_leaf(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_direct_typed_leaf(child, true, column_ids);
+ });
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY && contains_uuid_typed_value_field(field_schema)) {
+ return false;
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool can_use_direct_typed_only_value(const FieldSchema& variant_field,
+ const std::set<uint64_t>& column_ids) {
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ return (value_idx < 0 || !has_selected_column(variant_field.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ has_selected_direct_typed_leaf(variant_field.children[typed_value_idx], false,
+ column_ids) &&
+ can_direct_read_typed_value(variant_field.children[typed_value_idx], false, column_ids);
+}
+
+static DataTypePtr make_variant_struct_reader_type(const FieldSchema& field) {
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field.children.size());
+ child_names.reserve(field.children.size());
+ for (const auto& child : field.children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ return std::make_shared<DataTypeStruct>(child_types, child_names);
+}
+
+static ColumnPtr make_variant_struct_read_column(const FieldSchema& field,
+ const DataTypePtr& variant_struct_type) {
+ if (field.data_type->is_nullable()) {
+ return make_nullable(variant_struct_type)->create_column();
+ }
+ return variant_struct_type->create_column();
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static void fill_variant_leaf_type_info(const DataTypePtr& data_type, FieldWithDataType* value) {
+ auto leaf_type = remove_nullable(data_type);
+ while (leaf_type->get_primitive_type() == TYPE_ARRAY) {
+ leaf_type = remove_nullable(
+ assert_cast<const DataTypeArray*>(leaf_type.get())->get_nested_type());
+ }
+ if (is_decimal(leaf_type->get_primitive_type())) {
+ value->precision = leaf_type->get_precision();
+ value->scale = leaf_type->get_scale();
+ }
+}
+
+template <PrimitiveType Primitive>
+static Status fill_floating_point_variant_field(const Field& field, FieldWithDataType* value) {
+ const auto typed_value = field.get<Primitive>();
+ if (!std::isfinite(typed_value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point typed_value is not supported");
+ }
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+}
+
+static bool is_uuid_typed_value_field(const FieldSchema& field_schema) {
+ return field_schema.parquet_schema.__isset.logicalType &&
+ field_schema.parquet_schema.logicalType.__isset.UUID;
+}
+
+static bool contains_uuid_typed_value_field(const FieldSchema& field_schema) {
+ return is_uuid_typed_value_field(field_schema) ||
+ std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_uuid_typed_value_field(child); });
+}
+
+static Status uuid_field_to_string(const Field& field, std::string* uuid) {
+ StringRef bytes;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ bytes = StringRef(field.get<TYPE_STRING>());
+ break;
+ case TYPE_CHAR:
+ bytes = StringRef(field.get<TYPE_CHAR>());
+ break;
+ case TYPE_VARCHAR:
+ bytes = StringRef(field.get<TYPE_VARCHAR>());
+ break;
+ case TYPE_VARBINARY:
+ bytes = field.get<TYPE_VARBINARY>().to_string_ref();
+ break;
+ default:
+ return Status::Corruption("Parquet VARIANT UUID typed_value has unexpected Doris type {}",
+ field.get_type_name());
+ }
+ if (bytes.size != 16) {
+ return Status::Corruption("Parquet VARIANT UUID typed_value has invalid length {}",
+ bytes.size);
+ }
+ *uuid = parquet::format_variant_uuid(reinterpret_cast<const uint8_t*>(bytes.data));
+ return Status::OK();
+}
+
+static Status fill_uuid_variant_field(const Field& field, FieldWithDataType* value) {
+ std::string uuid;
+ RETURN_IF_ERROR(uuid_field_to_string(field, &uuid));
+ value->field = Field::create_field<TYPE_STRING>(std::move(uuid));
+ value->base_scalar_type_id = TYPE_STRING;
+ return Status::OK();
+}
+
+static uint8_t direct_array_dimensions(const DataTypePtr& data_type) {
+ uint8_t num_dimensions = 0;
+ auto type = remove_nullable(data_type);
+ while (type->get_primitive_type() == TYPE_ARRAY) {
+ ++num_dimensions;
+ type = remove_nullable(assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return num_dimensions;
+}
+
+static Status convert_uuid_array_value(const FieldSchema& field_schema, const Field& field,
+ Field* converted) {
+ if (field.is_null()) {
+ *converted = Field();
+ return Status::OK();
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ if (field_schema.children.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT UUID array typed_value has no element schema");
+ }
+ Array converted_elements;
+ const auto& elements = field.get<TYPE_ARRAY>();
+ converted_elements.reserve(elements.size());
+ for (const auto& element : elements) {
+ Field converted_element;
+ RETURN_IF_ERROR(convert_uuid_array_value(field_schema.children[0], element,
+ &converted_element));
+ converted_elements.push_back(std::move(converted_element));
+ }
+ *converted = Field::create_field<TYPE_ARRAY>(std::move(converted_elements));
+ return Status::OK();
+ }
+
+ if (!is_uuid_typed_value_field(field_schema)) {
+ return Status::Corruption("Parquet VARIANT UUID array has non-UUID element schema");
+ }
+ FieldWithDataType value;
+ RETURN_IF_ERROR(fill_uuid_variant_field(field, &value));
+ *converted = std::move(value.field);
+ return Status::OK();
+}
+
+static Status fill_uuid_array_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ RETURN_IF_ERROR(convert_uuid_array_value(field_schema, field, &value->field));
+ value->base_scalar_type_id = TYPE_STRING;
+ value->num_dimensions = direct_array_dimensions(field_schema.data_type);
+ return Status::OK();
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ if (is_uuid_typed_value_field(field_schema)) {
+ return fill_uuid_variant_field(field, value);
+ }
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ fill_variant_leaf_type_info(type, value);
+ return Status::OK();
+ case TYPE_FLOAT:
+ return fill_floating_point_variant_field<TYPE_FLOAT>(field, value);
+ case TYPE_DOUBLE:
+ return fill_floating_point_variant_field<TYPE_DOUBLE>(field, value);
+ case TYPE_TIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>())));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATE:
+ value->field = Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATE>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIME:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIME>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATEV2:
+ value->field =
+ Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIMEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present);
+
+static Status serialize_field_to_json(const DataTypePtr& data_type, const Field& field,
+ std::string* json) {
+ MutableColumnPtr column = data_type->create_column();
+ column->insert(field);
+
+ auto json_column = ColumnString::create();
+ VectorBufferWriter writer(*json_column);
+ auto serde = data_type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ RETURN_IF_ERROR(serde->serialize_one_cell_to_json(*column, 0, writer, options));
+ writer.commit();
+ *json = json_column->get_data_at(0).to_string();
+ return Status::OK();
+}
+
+static Status scalar_typed_value_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(field_schema, field, &value, present));
+ if (!*present) {
+ return Status::OK();
+ }
+ if (value.field.is_null()) {
+ *json = "null";
+ return Status::OK();
+ }
+ if (!is_uuid_typed_value_field(field_schema) &&
+ remove_nullable(field_schema.data_type)->get_primitive_type() == TYPE_VARBINARY) {
+ return Status::NotSupported(
+ "Parquet VARIANT binary typed_value cannot be serialized to JSON");
+ }
+
+ DataTypePtr json_type;
+ if (value.base_scalar_type_id != PrimitiveType::INVALID_TYPE) {
+ json_type = DataTypeFactory::instance().create_data_type(value.base_scalar_type_id, false,
+ value.precision, value.scale);
+ } else {
+ json_type = remove_nullable(field_schema.data_type);
+ }
+ return serialize_field_to_json(json_type, value.field, json);
+}
+
+static Status resolve_variant_metadata(const FieldSchema& variant_field, const Struct& fields,
+ const std::string* inherited_metadata, std::string* metadata,
+ bool* has_metadata) {
+ *has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ *metadata = *inherited_metadata;
+ *has_metadata = true;
+ }
+
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], metadata, &metadata_present));
+ *has_metadata = metadata_present;
+ }
+ return Status::OK();
+}
+
+static Status variant_typed_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, std::string* typed_json,
+ bool* typed_present) {
+ *typed_present = false;
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ if (typed_value_idx < 0) {
+ return Status::OK();
+ }
+ return typed_value_to_json(variant_field.children[typed_value_idx], fields[typed_value_idx],
+ metadata, typed_json, typed_present);
+}
+
+static Status variant_residual_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, bool has_metadata,
+ std::string* value_json, bool* value_present) {
+ *value_present = false;
+ const int value_idx = find_child_idx(variant_field, "value");
+ if (value_idx < 0) {
+ return Status::OK();
+ }
+
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, value_present));
+ if (!*value_present) {
+ return Status::OK();
+ }
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ return parquet::decode_variant_to_json(StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), value_json);
+}
+
+static Status merge_variant_value_and_typed_json(const std::string& value_json,
+ const std::string& typed_json, std::string* json) {
+ VariantMap value_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, PathInData(), &value_values));
+ VariantMap typed_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(typed_json, PathInData(), &typed_values));
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto root_value = value_values.find(PathInData());
+ if (root_value != value_values.end() && !is_empty_object_marker(root_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, PathInData()));
+ value_values.merge(std::move(typed_values));
+ return variant_map_to_json(std::move(value_values), json);
+}
+
+static Status variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ std::string metadata;
+ bool has_metadata = false;
+ RETURN_IF_ERROR(resolve_variant_metadata(variant_field, fields, inherited_metadata, &metadata,
+ &has_metadata));
+
+ std::string typed_json;
+ bool typed_present = false;
+ RETURN_IF_ERROR(variant_typed_value_to_json(variant_field, fields, metadata, &typed_json,
+ &typed_present));
+
+ std::string value_json;
+ bool value_present = false;
+ RETURN_IF_ERROR(variant_residual_value_to_json(variant_field, fields, metadata, has_metadata,
+ &value_json, &value_present));
+
+ if (value_present && typed_present) {
+ RETURN_IF_ERROR(merge_variant_value_and_typed_json(value_json, typed_json, json));
+ *present = true;
+ return Status::OK();
+ }
+
+ if (typed_present) {
+ *json = std::move(typed_json);
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+
+ *present = false;
+ return Status::OK();
+}
+
+static Status shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json, bool* present,
+ bool allow_scalar_typed_value_only_wrapper) {
+ if (is_variant_wrapper_field(field_schema, allow_scalar_typed_value_only_wrapper)) {
+ return variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ if (is_value_only_variant_wrapper_candidate(field_schema)) {
+ Status st = variant_to_json(field_schema, field, &metadata, json, present);
+ if (st.ok()) {
+ return st;
+ }
+ if (!st.is<ErrorCode::CORRUPTION>()) {
+ return st;
+ }
+ }
+ return typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+static Status typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ json->clear();
+ json->push_back('[');
+ for (size_t i = 0; i < elements.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(element_schema, elements[i], metadata, &element_json,
+ &element_present, true));
+ if (element_present) {
+ json->append(element_json);
+ } else {
+ json->append("null");
+ }
+ }
+ json->push_back(']');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ json->clear();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(typed_value_field.children[i], fields[i], metadata,
+ &child_json, &child_present, false));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ first = false;
+ }
+ json->push_back('}');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ switch (typed_type->get_primitive_type()) {
+ case TYPE_STRUCT:
+ return typed_struct_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_ARRAY:
+ return typed_array_to_json(typed_value_field, field, metadata, json, present);
+ default:
+ return scalar_typed_value_to_json(typed_value_field, field, json, present);
+ }
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values);
+
+static Status variant_to_variant_map(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ VariantMap value_values;
+ bool value_present = false;
+ const PathInData current_path = path->build();
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ RETURN_IF_ERROR(parquet::decode_variant_to_variant_map(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), current_path, &value_values,
+ string_values));
+ }
+ }
+
+ VariantMap typed_values;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_variant_map(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, path,
+ &typed_values, &typed_present, string_values));
+ }
+
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto current_value = value_values.find(current_path);
+ if (value_present && typed_present && current_value != value_values.end() &&
+ !is_empty_object_marker(current_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, current_path));
+ values->merge(std::move(value_values));
+ values->merge(std::move(typed_values));
+ *present = value_present || typed_present;
+ return Status::OK();
+}
+
+static Status shredded_field_to_variant_map(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values) {
+ if (is_variant_wrapper_field(field_schema, false)) {
+ return variant_to_variant_map(field_schema, field, &metadata, path, values, present,
+ string_values);
+ }
+ if (is_value_only_variant_wrapper_candidate(field_schema)) {
+ Status st = variant_to_variant_map(field_schema, field, &metadata, path, values, present,
+ string_values);
+ if (st.ok()) {
+ return st;
+ }
+ if (!st.is<ErrorCode::CORRUPTION>()) {
+ return st;
+ }
+ }
+ return typed_value_to_variant_map(field_schema, field, metadata, path, values, present,
+ string_values);
+}
+
+static Status append_typed_field_to_variant_map(const FieldSchema& typed_value_field,
+ const Field& field, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static Status typed_array_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>*) {
+ if (contains_uuid_typed_value_field(typed_value_field) &&
+ is_direct_variant_leaf_type(typed_value_field.data_type)) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(fill_uuid_array_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+ }
+ if (is_direct_variant_leaf_type(typed_value_field.data_type)) {
+ return append_typed_field_to_variant_map(typed_value_field, field, path, values, present);
+ }
+
+ std::string value_json;
+ RETURN_IF_ERROR(typed_value_to_json(typed_value_field, field, metadata, &value_json, present));
+ if (*present) {
Review Comment:
This row-wise typed array path still serializes non-direct typed arrays through JSON, which rejects valid typed binary leaves nested inside array elements. A typed-only VARIANT such as `typed_value: array<struct<payload: binary>>` cannot use the direct leaf path, so full projection calls `typed_value_to_json()` for the array; the nested `payload` then reaches `scalar_typed_value_to_json()` and returns `NotSupported("Parquet VARIANT binary typed_value cannot be serialized to JSON")`. Valid typed binary data should be preserved just like the object-field test does outside arrays. This is separate from the existing decimal array type-fidelity issue because it is a hard failure for binary leaves inside complex typed arrays. Please build the array `VariantMap` recursively without a JSON round trip, or otherwise handle binary nested leaves, and add a regression/unit case for `array<struct<varbinary>>`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253571052
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java:
##########
@@ -313,6 +342,14 @@ public Void visitElementAt(ElementAt elementAt, CollectorContext context) {
return continueCollectAccessPath(first, context);
}
return visit(elementAt, context);
+ } else if (first.getDataType().isVariantType() && arguments.size() >= 2) {
+ CollectorContext variantRootContext = context.copy();
+ variantRootContext.setCollectVariantRoot(true);
Review Comment:
This still does not force a full VARIANT read when the dynamic subscript has an outer suffix. For `v[cast(id AS string)]['x']`, the outer literal access leaves `accessPathBuilder = [x]`; this branch sets `collectVariantRoot`, but `visitSlotReference()` checks the non-empty builder first and records `[v, x]` instead of a root `[v]` demand. Since the dynamic key can select any top-level field before applying `['x']`, pruning to only `x` can drop required data and return missing/null values when combined with a sibling predicate such as `v['k'] IS NOT NULL`. Please clear/override the suffix when a dynamic VARIANT subscript requires a root read, or make `collectVariantRoot` take precedence over a non-empty builder, and add coverage for a chained dynamic subscript plus a sibling nested predicate.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469437132
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470535235
# BE Regression && UT Coverage Report
Increment line coverage `77.54% (2503/3228)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_b7402399a565dc0a3e621ecb4d61cfd48a4eb2fc_merge/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_b7402399a565dc0a3e621ecb4d61cfd48a4eb2fc_merge/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 71.96% (27330/37982) |
| Line Coverage | 55.38% (291682/526699) |
| Region Coverage | 52.33% (241929/462341) |
| Branch Coverage | 53.69% (104760/195104) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466909970
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466910126
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467589579
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467599286
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468048419
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468734643
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253691117
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +2645,132 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _column_ids = column_ids;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypePtr variant_struct_type = make_variant_struct_reader_type(*field);
+ _variant_struct_field->data_type = variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+// NOLINTNEXTLINE(readability-function-size): existing variant read path stays local to avoid churn.
+Status VariantColumnReader::read_column_data(
+ ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
+ (void)root_node;
+ if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_VARIANT) {
+ return Status::Corruption(
+ "Wrong data type for column '{}', expected Variant type, actual type: {}.",
+ _field_schema->name, type->get_name());
+ }
+
+ const auto& variant_struct_type = _variant_struct_field->data_type;
+ ColumnPtr struct_column = make_variant_struct_read_column(*_field_schema, variant_struct_type);
+ const size_t old_struct_rows = struct_column->size();
+ auto const_node = TableSchemaChangeHelper::ConstNode::get_instance();
+ RETURN_IF_ERROR(_struct_reader->read_column_data(struct_column, variant_struct_type, const_node,
+ filter_map, batch_size, read_rows, eof,
+ is_dict_filter, real_column_size));
+
+ const size_t new_struct_rows = struct_column->size() - old_struct_rows;
+ if (new_struct_rows == 0) {
+ return Status::OK();
+ }
+
+ MutableColumnPtr variant_column_ptr;
+ NullMap* null_map_ptr = nullptr;
+ auto mutable_column = doris_column->assume_mutable();
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = assert_cast<ColumnNullable*>(mutable_column.get());
+ variant_column_ptr = nullable_column->get_nested_column_ptr();
+ null_map_ptr = &nullable_column->get_null_map_data();
+ } else {
+ if (_field_schema->data_type->is_nullable()) {
+ return Status::Corruption("Not nullable column has null values in parquet file");
+ }
+ variant_column_ptr = std::move(mutable_column);
+ }
+ auto* variant_column = assert_cast<ColumnVariant*>(variant_column_ptr.get());
+
+ const IColumn* variant_struct_source = struct_column.get();
Review Comment:
The direct typed-only path does not materialize a root VARIANT null when the selected complex `typed_value` itself is null. For a typed-only schema like `metadata, optional group typed_value { optional int64 a; }`, `can_use_direct_typed_only_value()` is true, this call records the nullable `typed_value` null map only as a parent null, and `append_direct_typed_empty_object_markers()` skips the marker for that row. The SQL null map below is populated only from the outer VARIANT struct null map, so the row is returned as a non-SQL-null VARIANT with no root null marker/empty payload instead of the row-wise behavior at lines 2759-2760, which inserts `PathInData()` with `Field()` when no value or typed value is present. Please make the direct path emit the same root VARIANT null for absent complex `typed_value` rows (or fall back to row-wise for this case), and add coverage for typed-only optional struct/list/map `typed_value` null rows.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468285348
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469361346
<details>
<summary>TPC-DS: <b>Total hot run time: 168985 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 8587400dc65063b8e8ea52ca286f3e7831788e4a, data reload: false
query5 4322 654 509 509
query6 340 221 204 204
query7 4225 606 315 315
query8 335 245 218 218
query9 8828 3965 3961 3961
query10 446 327 294 294
query11 5775 2410 2227 2227
query12 192 130 125 125
query13 1333 609 401 401
query14 5932 5344 5075 5075
query14_1 4310 4321 4325 4321
query15 209 200 183 183
query16 1000 453 418 418
query17 943 706 592 592
query18 2424 491 344 344
query19 210 193 163 163
query20 133 132 128 128
query21 209 142 126 126
query22 13539 13553 13478 13478
query23 17074 16446 16017 16017
query23_1 16251 16102 16264 16102
query24 7516 1772 1340 1340
query24_1 1301 1299 1303 1299
query25 584 500 447 447
query26 1335 322 181 181
query27 2656 569 335 335
query28 4491 1922 1925 1922
query29 1014 645 517 517
query30 304 243 198 198
query31 1119 1074 944 944
query32 93 87 76 76
query33 554 359 318 318
query34 1160 1153 638 638
query35 770 801 694 694
query36 1336 1352 1206 1206
query37 155 106 95 95
query38 3218 3165 3054 3054
query39 940 912 904 904
query39_1 871 888 881 881
query40 241 152 132 132
query41 74 70 70 70
query42 114 111 112 111
query43 323 332 285 285
query44
query45 216 208 203 203
query46 1101 1243 752 752
query47 2306 2309 2201 2201
query48 416 419 296 296
query49 657 519 408 408
query50 969 347 275 275
query51 4355 4286 4238 4238
query52 110 108 97 97
query53 254 284 209 209
query54 327 281 278 278
query55 96 94 92 92
query56 312 328 320 320
query57 1410 1416 1319 1319
query58 319 281 281 281
query59 1620 1643 1458 1458
query60 370 319 304 304
query61 160 151 156 151
query62 676 631 537 537
query63 242 198 204 198
query64 2413 793 611 611
query65
query66 1722 467 372 372
query67 29987 29990 29770 29770
query68
query69 445 338 313 313
query70 1032 1021 1016 1016
query71 309 268 264 264
query72 3001 2715 2386 2386
query73 869 738 421 421
query74 5043 4855 4726 4726
query75 2653 2600 2280 2280
query76 2304 1118 770 770
query77 393 396 337 337
query78 12047 12025 11526 11526
query79 1456 1061 764 764
query80 640 533 444 444
query81 451 288 243 243
query82 1416 158 121 121
query83 364 279 247 247
query84 307 138 115 115
query85 872 530 443 443
query86 398 311 298 298
query87 3368 3385 3217 3217
query88 3508 2655 2617 2617
query89 425 393 336 336
query90 1963 170 176 170
query91 193 166 138 138
query92 79 80 69 69
query93 1459 1428 818 818
query94 531 359 315 315
query95 666 455 342 342
query96 1079 784 345 345
query97 2713 2686 2541 2541
query98 240 229 226 226
query99 1112 1107 982 982
Total cold run time: 252163 ms
Total hot run time: 168985 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467865446
<details>
<summary>TPC-H: <b>Total hot run time: 30850 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit c130ee160e3cb1a7e67aa06658c0224b248413dc, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17621 3918 3839 3839
q2 q3 10804 1370 817 817
q4 4682 477 361 361
q5 7556 2205 2110 2110
q6 257 176 134 134
q7 938 800 614 614
q8 9342 1716 1549 1549
q9 6622 4939 4829 4829
q10 6451 2121 1780 1780
q11 443 267 241 241
q12 690 423 293 293
q13 18203 3410 2792 2792
q14 275 257 240 240
q15 q16 818 793 711 711
q17 1023 893 942 893
q18 6818 5830 5617 5617
q19 1274 1254 1062 1062
q20 525 408 254 254
q21 5789 2614 2411 2411
q22 415 358 303 303
Total cold run time: 100546 ms
Total hot run time: 30850 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4241 4147 4111 4111
q2 q3 4476 4901 4330 4330
q4 2127 2200 1409 1409
q5 4359 4278 4291 4278
q6 231 175 129 129
q7 2189 1785 1745 1745
q8 2509 2119 2045 2045
q9 7878 7811 7734 7734
q10 4513 4474 4079 4079
q11 579 410 378 378
q12 908 775 521 521
q13 3323 3575 3018 3018
q14 305 307 278 278
q15 q16 707 730 656 656
q17 1357 1300 1334 1300
q18 7915 7401 7206 7206
q19 1092 1086 1112 1086
q20 2215 2209 1944 1944
q21 5364 4630 4436 4436
q22 521 452 414 414
Total cold run time: 56809 ms
Total hot run time: 51097 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468022389
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468483351
<details>
<summary>TPC-DS: <b>Total hot run time: 168847 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 7cbca3a936be42996b737e56c61ce4c1658ffacc, data reload: false
query5 4348 647 513 513
query6 336 217 196 196
query7 4329 548 301 301
query8 328 239 221 221
query9 8806 3994 3960 3960
query10 446 360 300 300
query11 5788 2384 2149 2149
query12 181 128 123 123
query13 1283 583 408 408
query14 5953 5391 5044 5044
query14_1 4379 4396 4345 4345
query15 208 204 182 182
query16 1003 473 449 449
query17 1164 747 600 600
query18 2459 505 366 366
query19 220 234 181 181
query20 140 131 132 131
query21 212 145 125 125
query22 13587 13598 13308 13308
query23 17289 16343 15998 15998
query23_1 16277 16270 16220 16220
query24 7492 1770 1305 1305
query24_1 1295 1310 1295 1295
query25 588 513 425 425
query26 1324 314 175 175
query27 2692 574 342 342
query28 4472 1953 1924 1924
query29 1004 614 493 493
query30 305 235 202 202
query31 1109 1088 926 926
query32 92 75 71 71
query33 536 349 287 287
query34 1181 1119 633 633
query35 761 776 663 663
query36 1343 1399 1165 1165
query37 158 101 87 87
query38 3208 3144 3072 3072
query39 922 921 891 891
query39_1 881 911 889 889
query40 235 146 122 122
query41 65 62 65 62
query42 110 109 109 109
query43 323 333 290 290
query44
query45 218 201 192 192
query46 1046 1153 759 759
query47 2324 2350 2196 2196
query48 382 420 300 300
query49 636 490 382 382
query50 966 350 252 252
query51 4356 4306 4220 4220
query52 106 108 99 99
query53 251 276 201 201
query54 328 269 247 247
query55 92 89 86 86
query56 291 308 300 300
query57 1415 1440 1329 1329
query58 292 276 260 260
query59 1561 1721 1457 1457
query60 318 316 309 309
query61 159 153 159 153
query62 664 625 569 569
query63 240 204 210 204
query64 2449 789 644 644
query65
query66 1782 470 369 369
query67 30089 29977 29814 29814
query68
query69 452 351 299 299
query70 1018 971 984 971
query71 312 271 269 269
query72 3101 2655 2409 2409
query73 841 731 407 407
query74 5048 4886 4746 4746
query75 2660 2581 2246 2246
query76 2302 1119 761 761
query77 394 404 314 314
query78 12136 12211 11691 11691
query79 1346 996 730 730
query80 617 529 448 448
query81 445 279 239 239
query82 497 162 122 122
query83 356 282 247 247
query84 264 144 108 108
query85 873 538 446 446
query86 406 335 314 314
query87 3410 3358 3223 3223
query88 3451 2645 2636 2636
query89 434 381 334 334
query90 1942 184 181 181
query91 179 163 140 140
query92 78 77 74 74
query93 1503 1464 837 837
query94 538 357 321 321
query95 693 392 356 356
query96 1027 806 342 342
query97 2740 2703 2610 2610
query98 238 230 229 229
query99 1088 1100 981 981
Total cold run time: 252136 ms
Total hot run time: 168847 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468462598
# FE UT Coverage Report
Increment line coverage ` 73.05% (103/141)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_3853dff1ae1dd95a975ee64df294feab76fab9db/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_3853dff1ae1dd95a975ee64df294feab76fab9db/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3254086484
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java:
##########
@@ -1058,13 +1058,23 @@ public TFileFormatType getFileFormatType() throws UserException {
if (icebergFormat.equalsIgnoreCase("parquet")) {
type = TFileFormatType.FORMAT_PARQUET;
} else if (icebergFormat.equalsIgnoreCase("orc")) {
+ validateVariantReadSupported(icebergFormat);
type = TFileFormatType.FORMAT_ORC;
} else {
throw new DdlException(String.format("Unsupported format name: %s for iceberg table.", icebergFormat));
}
return type;
}
+ private void validateVariantReadSupported(String icebergFormat) throws DdlException {
+ for (SlotDescriptor slot : desc.getSlots()) {
+ if (slot.getColumn().getType().isVariantType()) {
Review Comment:
This validation only checks whether the top-level selected column is VARIANT, but this PR maps Iceberg VARIANT recursively inside complex types. An ORC Iceberg schema such as `struct<s: variant>` or `array<variant>` will pass here because the slot column type is STRUCT/ARRAY, then proceed into an unsupported non-Parquet VARIANT read path. Please recursively inspect the selected column type for nested VARIANTs and add ORC Iceberg coverage for complex columns containing VARIANT.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +127,1823 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_unannotated_variant_value_field(const FieldSchema& field) {
+ // VARIANT residual value is raw binary; annotated strings named value are user fields.
+ return field.lower_case_name == "value" && field.physical_type == tparquet::Type::BYTE_ARRAY &&
+ !field.parquet_schema.__isset.logicalType &&
+ !field.parquet_schema.__isset.converted_type;
+}
+
+static bool is_unannotated_variant_metadata_field(const FieldSchema& field) {
+ return field.lower_case_name == "metadata" &&
+ field.physical_type == tparquet::Type::BYTE_ARRAY &&
+ !field.parquet_schema.__isset.logicalType &&
+ !field.parquet_schema.__isset.converted_type;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (!is_unannotated_variant_metadata_field(child)) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (!is_unannotated_variant_value_field(child)) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata) {
+ return type->get_primitive_type() == TYPE_VARIANT && (has_value || typed_value != nullptr);
+ }
+ if (has_value) {
+ return typed_value != nullptr;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static bool is_value_only_variant_wrapper_candidate(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_value = false;
+ for (const auto& child : field.children) {
+ if (is_unannotated_variant_value_field(child)) {
+ has_value = true;
+ continue;
+ }
+ return false;
+ }
+ return has_value;
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status make_jsonb_field(std::string_view json, FieldWithDataType* value) {
+ JsonBinaryValue jsonb_value;
+ RETURN_IF_ERROR(jsonb_value.from_json_string(json.data(), json.size()));
+ value->field =
+ Field::create_field<TYPE_JSONB>(JsonbField(jsonb_value.value(), jsonb_value.size()));
+ value->base_scalar_type_id = TYPE_JSONB;
+ value->num_dimensions = 0;
+ value->precision = 0;
+ value->scale = 0;
+ return Status::OK();
+}
+
+static std::string make_null_array_json(size_t elements) {
+ std::string json = "[";
+ for (size_t i = 0; i < elements; ++i) {
+ if (i != 0) {
+ json.push_back(',');
+ }
+ json.append("null");
+ }
+ json.push_back(']');
+ return json;
+}
+
+static Status make_empty_object_field(Field* field) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field("{}", &value));
+ *field = std::move(value.field);
+ return Status::OK();
+}
+
+static Status insert_jsonb_value(const PathInData& path, std::string_view json,
+ VariantMap* values) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field(json, &value));
+ (*values)[path] = std::move(value);
+ return Status::OK();
+}
+
+static Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ return insert_jsonb_value(path, "{}", values);
+}
+
+static bool is_empty_object_marker(const FieldWithDataType& value) {
+ if (value.field.get_type() != TYPE_JSONB) {
+ return false;
+ }
+ const auto& jsonb = value.field.get<TYPE_JSONB>();
+ const JsonbDocument* document = nullptr;
+ Status st =
+ JsonbDocument::checkAndCreateDocument(jsonb.get_value(), jsonb.get_size(), &document);
+ if (!st.ok() || document == nullptr || document->getValue() == nullptr ||
+ !document->getValue()->isObject()) {
+ return false;
+ }
+ return document->getValue()->unpack<ObjectVal>()->numElem() == 0;
+}
+
+static Status collect_empty_object_markers(const rapidjson::Value& value, PathInDataBuilder* path,
+ VariantMap* values) {
+ if (!value.IsObject()) {
+ return Status::OK();
+ }
+ if (value.MemberCount() == 0) {
+ return insert_empty_object_marker(path->build(), values);
+ }
+ for (auto it = value.MemberBegin(); it != value.MemberEnd(); ++it) {
+ if (it->value.IsObject()) {
+ path->append(std::string_view(it->name.GetString(), it->name.GetStringLength()), false);
+ RETURN_IF_ERROR(collect_empty_object_markers(it->value, path, values));
+ path->pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+static Status add_empty_object_markers_from_json(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ if (json.find("{}") == std::string::npos) {
+ return Status::OK();
+ }
+ rapidjson::Document document;
+ document.Parse(json.data(), json.size());
+ if (document.HasParseError()) {
+ return Status::Corruption("Invalid Parquet VARIANT decoded JSON");
+ }
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ return collect_empty_object_markers(document, &path, values);
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (!parsed.is_null()) {
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ }
+ RETURN_IF_ERROR(add_empty_object_markers_from_json(json, prefix, values));
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static bool path_has_prefix(const PathInData& path, const PathInData& prefix) {
+ const auto& parts = path.get_parts();
+ const auto& prefix_parts = prefix.get_parts();
+ if (parts.size() < prefix_parts.size()) {
+ return false;
+ }
+ for (size_t i = 0; i < prefix_parts.size(); ++i) {
+ if (parts[i] != prefix_parts[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool has_descendant_path(const VariantMap& values, const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ return std::ranges::any_of(values, [&](const auto& entry) {
+ const auto& path = entry.first;
+ return path.get_parts().size() > prefix_size && path_has_prefix(path, prefix);
+ });
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* values,
+ const VariantMap& shadowing_values) {
+ for (auto it = values->begin(); it != values->end();) {
+ if (is_empty_object_marker(it->second) &&
+ (has_descendant_path(*values, it->first) ||
+ has_descendant_path(shadowing_values, it->first))) {
+ it = values->erase(it);
+ continue;
+ }
+ ++it;
+ }
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* value_values,
+ VariantMap* typed_values) {
+ erase_shadowed_empty_object_markers(value_values, *typed_values);
+ erase_shadowed_empty_object_markers(typed_values, *value_values);
+}
+
+static Status check_no_shredded_value_typed_duplicates(const VariantMap& value_values,
+ const VariantMap& typed_values,
+ const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ for (const auto& value_entry : value_values) {
+ const auto& value_path = value_entry.first;
+ if (!path_has_prefix(value_path, prefix)) {
+ continue;
+ }
+ if (value_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(value_entry.second) &&
+ !has_descendant_path(typed_values, value_path)) {
+ continue;
+ }
+ if (!typed_values.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT residual value conflicts with typed_value at path {}",
+ value_path.get_path());
+ }
+ continue;
+ }
+ for (const auto& typed_entry : typed_values) {
+ const auto& typed_path = typed_entry.first;
+ if (!path_has_prefix(typed_path, prefix)) {
+ continue;
+ }
+ if (typed_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(typed_entry.second) &&
+ !has_descendant_path(value_values, typed_path)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ if (value_path.get_parts()[prefix_size] == typed_path.get_parts()[prefix_size]) {
+ if (value_path == typed_path && is_empty_object_marker(value_entry.second) &&
+ is_empty_object_marker(typed_entry.second)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+static bool has_direct_typed_parent_null(const std::vector<const NullMap*>& null_maps, size_t row) {
+ return std::ranges::any_of(null_maps, [&](const NullMap* null_map) {
+ DCHECK_LT(row, null_map->size());
+ return (*null_map)[row];
+ });
+}
+
+static void insert_direct_typed_leaf_range(const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps,
+ IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ nullable_leaf.get_nested_column().insert_range_from(*value_column, start, rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static bool is_temporal_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_floating_point_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_uuid_typed_value_field(const FieldSchema& field_schema);
+static bool contains_uuid_typed_value_field(const FieldSchema& field_schema);
+
+static DataTypePtr direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return std::make_shared<DataTypeInt64>();
+ }
+ return type;
+}
+
+static DataTypePtr direct_variant_leaf_type(const FieldSchema& field_schema) {
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (is_uuid_typed_value_field(field_schema)) {
+ return std::make_shared<DataTypeString>();
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ DORIS_CHECK(!field_schema.children.empty());
+ DataTypePtr nested_type = direct_variant_leaf_type(field_schema.children[0]);
+ if (field_schema.children[0].data_type->is_nullable()) {
+ nested_type = make_nullable(nested_type);
+ }
+ return std::make_shared<DataTypeArray>(nested_type);
+ }
+ return direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool contains_temporal_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_temporal_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static bool contains_floating_point_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_floating_point_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_floating_point_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static int64_t direct_temporal_variant_value(PrimitiveType type, const IColumn& column,
+ size_t row) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ return static_cast<int64_t>(
+ std::llround(assert_cast<const ColumnTimeV2&>(column).get_data()[row]));
+ case TYPE_DATE:
+ return variant_date_value(assert_cast<const ColumnDate&>(column).get_data()[row]);
+ case TYPE_DATETIME:
+ return variant_datetime_value(assert_cast<const ColumnDateTime&>(column).get_data()[row]);
+ case TYPE_DATEV2:
+ return variant_date_value(assert_cast<const ColumnDateV2&>(column).get_data()[row]);
+ case TYPE_DATETIMEV2:
+ return variant_datetime_value(assert_cast<const ColumnDateTimeV2&>(column).get_data()[row]);
+ case TYPE_TIMESTAMPTZ:
+ return variant_datetime_value(
+ assert_cast<const ColumnTimeStampTz&>(column).get_data()[row]);
+ default:
+ DORIS_CHECK(false);
+ return 0;
+ }
+}
+
+static void insert_direct_typed_temporal_leaf_range(
+ PrimitiveType type, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnInt64&>(nullable_leaf.get_nested_column()).get_data();
+ data.reserve(data.size() + rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ data.push_back(direct_temporal_variant_value(type, *value_column, row));
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static Status insert_direct_typed_uuid_leaf_range(
+ const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnString&>(nullable_leaf.get_nested_column());
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ const bool is_null = leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row);
+ if (is_null) {
+ data.insert_default();
+ null_map.push_back(1);
+ continue;
+ }
+ StringRef bytes = value_column->get_data_at(row);
+ if (bytes.size != 16) {
+ return Status::Corruption("Parquet VARIANT UUID typed_value has invalid length {}",
+ bytes.size);
+ }
+ std::string uuid =
+ parquet::format_variant_uuid(reinterpret_cast<const uint8_t*>(bytes.data));
+ data.insert_data(uuid.data(), uuid.size());
+ null_map.push_back(0);
+ }
+ return Status::OK();
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY:
+ return true;
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool has_selected_direct_typed_leaf(const FieldSchema& field_schema,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return false;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ return has_selected_direct_typed_leaf(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_direct_typed_leaf(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool can_use_direct_typed_only_value(const FieldSchema& variant_field,
+ const std::set<uint64_t>& column_ids) {
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ return (value_idx < 0 || !has_selected_column(variant_field.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ has_selected_direct_typed_leaf(variant_field.children[typed_value_idx], false,
+ column_ids) &&
+ can_direct_read_typed_value(variant_field.children[typed_value_idx], false, column_ids);
+}
+
+static DataTypePtr make_variant_struct_reader_type(const FieldSchema& field) {
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field.children.size());
+ child_names.reserve(field.children.size());
+ for (const auto& child : field.children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ return std::make_shared<DataTypeStruct>(child_types, child_names);
+}
+
+static ColumnPtr make_variant_struct_read_column(const FieldSchema& field,
+ const DataTypePtr& variant_struct_type) {
+ if (field.data_type->is_nullable()) {
+ return make_nullable(variant_struct_type)->create_column();
+ }
+ return variant_struct_type->create_column();
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static void fill_variant_leaf_type_info(const DataTypePtr& data_type, FieldWithDataType* value) {
+ auto leaf_type = remove_nullable(data_type);
+ size_t num_dimensions = 0;
+ while (leaf_type->get_primitive_type() == TYPE_ARRAY) {
+ ++num_dimensions;
+ leaf_type = remove_nullable(
+ assert_cast<const DataTypeArray*>(leaf_type.get())->get_nested_type());
+ }
+ DCHECK_LE(num_dimensions, std::numeric_limits<uint8_t>::max());
+ if (value->base_scalar_type_id == INVALID_TYPE) {
+ value->base_scalar_type_id = leaf_type->get_primitive_type();
+ }
+ if (value->num_dimensions == 0 && num_dimensions > 0) {
+ value->num_dimensions = static_cast<uint8_t>(num_dimensions);
+ }
+ if (is_decimal(leaf_type->get_primitive_type())) {
+ value->precision = leaf_type->get_precision();
+ value->scale = leaf_type->get_scale();
+ }
+}
+
+static Status fill_floating_point_variant_field(const Field& field, FieldWithDataType* value) {
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+}
+
+static Status fill_floating_point_variant_field(PrimitiveType type, const Field& field,
+ FieldWithDataType* value) {
+ DORIS_CHECK(type == TYPE_FLOAT || type == TYPE_DOUBLE);
+ return fill_floating_point_variant_field(field, value);
+}
+
+static bool is_uuid_typed_value_field(const FieldSchema& field_schema) {
+ return field_schema.parquet_schema.__isset.logicalType &&
+ field_schema.parquet_schema.logicalType.__isset.UUID;
+}
+
+static bool contains_uuid_typed_value_field(const FieldSchema& field_schema) {
+ return is_uuid_typed_value_field(field_schema) ||
+ std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_uuid_typed_value_field(child); });
+}
+
+static Status uuid_field_to_string(const Field& field, std::string* uuid) {
+ StringRef bytes;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ bytes = StringRef(field.get<TYPE_STRING>());
+ break;
+ case TYPE_CHAR:
+ bytes = StringRef(field.get<TYPE_CHAR>());
+ break;
+ case TYPE_VARCHAR:
+ bytes = StringRef(field.get<TYPE_VARCHAR>());
+ break;
+ case TYPE_VARBINARY:
+ bytes = field.get<TYPE_VARBINARY>().to_string_ref();
+ break;
+ default:
+ return Status::Corruption("Parquet VARIANT UUID typed_value has unexpected Doris type {}",
+ field.get_type_name());
+ }
+ if (bytes.size != 16) {
+ return Status::Corruption("Parquet VARIANT UUID typed_value has invalid length {}",
+ bytes.size);
+ }
+ *uuid = parquet::format_variant_uuid(reinterpret_cast<const uint8_t*>(bytes.data));
+ return Status::OK();
+}
+
+static Status fill_uuid_variant_field(const Field& field, FieldWithDataType* value) {
+ std::string uuid;
+ RETURN_IF_ERROR(uuid_field_to_string(field, &uuid));
+ value->field = Field::create_field<TYPE_STRING>(std::move(uuid));
+ value->base_scalar_type_id = TYPE_STRING;
+ return Status::OK();
+}
+
+static Status fill_temporal_variant_field(PrimitiveType type, const Field& field,
+ FieldWithDataType* value) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>())));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATE:
+ value->field = Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATE>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIME:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIME>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATEV2:
+ value->field =
+ Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIMEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ default:
+ DORIS_CHECK(false);
+ return Status::OK();
+ }
+}
+
+static uint8_t direct_array_dimensions(const DataTypePtr& data_type) {
+ uint8_t num_dimensions = 0;
+ auto type = remove_nullable(data_type);
+ while (type->get_primitive_type() == TYPE_ARRAY) {
+ ++num_dimensions;
+ type = remove_nullable(assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return num_dimensions;
+}
+
+static PrimitiveType direct_array_base_scalar_type(const FieldSchema& field_schema) {
+ auto leaf_type = remove_nullable(direct_variant_leaf_type(field_schema));
+ while (leaf_type->get_primitive_type() == TYPE_ARRAY) {
+ leaf_type = remove_nullable(
+ assert_cast<const DataTypeArray*>(leaf_type.get())->get_nested_type());
+ }
+ return leaf_type->get_primitive_type();
+}
+
+static Status convert_direct_array_value(const FieldSchema& field_schema, const Field& field,
+ Field* converted) {
+ if (field.is_null()) {
+ *converted = Field();
+ return Status::OK();
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ if (field_schema.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+ Array converted_elements;
+ const auto& elements = field.get<TYPE_ARRAY>();
+ converted_elements.reserve(elements.size());
+ for (const auto& element : elements) {
+ Field converted_element;
+ RETURN_IF_ERROR(convert_direct_array_value(field_schema.children[0], element,
+ &converted_element));
+ converted_elements.push_back(std::move(converted_element));
+ }
+ *converted = Field::create_field<TYPE_ARRAY>(std::move(converted_elements));
+ return Status::OK();
+ }
+
+ if (is_uuid_typed_value_field(field_schema)) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(fill_uuid_variant_field(field, &value));
+ *converted = std::move(value.field);
+ return Status::OK();
+ }
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(fill_temporal_variant_field(type->get_primitive_type(), field, &value));
+ *converted = std::move(value.field);
+ return Status::OK();
+ }
+ if (is_floating_point_variant_leaf_type(type->get_primitive_type())) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(
+ fill_floating_point_variant_field(type->get_primitive_type(), field, &value));
+ *converted = std::move(value.field);
+ return Status::OK();
+ }
+
+ *converted = field;
+ return Status::OK();
+}
+
+static Status insert_direct_typed_array_leaf_range(
+ const FieldSchema& field_schema, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = nullable_leaf.get_nested_column();
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ const bool is_null = leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row);
+ if (is_null) {
+ data.insert_default();
+ null_map.push_back(1);
+ continue;
+ }
+
+ Field field;
+ value_column->get(row, field);
+ Field converted;
+ RETURN_IF_ERROR(convert_direct_array_value(field_schema, field, &converted));
+ data.insert(converted);
+ null_map.push_back(0);
+ }
+ return Status::OK();
+}
+
+static Status fill_direct_array_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ RETURN_IF_ERROR(convert_direct_array_value(field_schema, field, &value->field));
+ value->base_scalar_type_id = direct_array_base_scalar_type(field_schema);
+ value->num_dimensions = direct_array_dimensions(field_schema.data_type);
+ return Status::OK();
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ if (is_uuid_typed_value_field(field_schema)) {
+ return fill_uuid_variant_field(field, value);
+ }
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return fill_temporal_variant_field(type->get_primitive_type(), field, value);
+ }
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ fill_variant_leaf_type_info(type, value);
+ return Status::OK();
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ return fill_floating_point_variant_field(field, value);
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present);
+static Status typed_map_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values);
+
+static Status serialize_field_to_json(const DataTypePtr& data_type, const Field& field,
+ std::string* json) {
+ MutableColumnPtr column = data_type->create_column();
+ column->insert(field);
+
+ auto json_column = ColumnString::create();
+ VectorBufferWriter writer(*json_column);
+ auto serde = data_type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ RETURN_IF_ERROR(serde->serialize_one_cell_to_json(*column, 0, writer, options));
+ writer.commit();
+ *json = json_column->get_data_at(0).to_string();
+ return Status::OK();
+}
+
+static Status scalar_typed_value_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(field_schema, field, &value, present));
+ if (!*present) {
+ return Status::OK();
+ }
+ if (value.field.is_null()) {
+ *json = "null";
+ return Status::OK();
+ }
+ if (!is_uuid_typed_value_field(field_schema) &&
+ remove_nullable(field_schema.data_type)->get_primitive_type() == TYPE_VARBINARY) {
+ return Status::NotSupported(
+ "Parquet VARIANT binary typed_value cannot be serialized to JSON");
+ }
+
+ DataTypePtr json_type;
+ if (value.base_scalar_type_id != PrimitiveType::INVALID_TYPE) {
+ json_type = DataTypeFactory::instance().create_data_type(value.base_scalar_type_id, false,
+ value.precision, value.scale);
+ } else {
+ json_type = remove_nullable(field_schema.data_type);
+ }
+ return serialize_field_to_json(json_type, value.field, json);
+}
+
+static Status resolve_variant_metadata(const FieldSchema& variant_field, const Struct& fields,
+ const std::string* inherited_metadata, std::string* metadata,
+ bool* has_metadata) {
+ *has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ *metadata = *inherited_metadata;
+ *has_metadata = true;
+ }
+
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], metadata, &metadata_present));
+ *has_metadata = metadata_present;
+ }
+ return Status::OK();
+}
+
+static Status variant_typed_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, std::string* typed_json,
+ bool* typed_present) {
+ *typed_present = false;
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ if (typed_value_idx < 0) {
+ return Status::OK();
+ }
+ return typed_value_to_json(variant_field.children[typed_value_idx], fields[typed_value_idx],
+ metadata, typed_json, typed_present);
+}
+
+static Status variant_residual_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, bool has_metadata,
+ std::string* value_json, bool* value_present) {
+ *value_present = false;
+ const int value_idx = find_child_idx(variant_field, "value");
+ if (value_idx < 0) {
+ return Status::OK();
+ }
+
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, value_present));
+ if (!*value_present) {
+ return Status::OK();
+ }
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ return parquet::decode_variant_to_json(StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), value_json);
+}
+
+static Status merge_variant_value_and_typed_json(const std::string& value_json,
+ const std::string& typed_json, std::string* json) {
+ VariantMap value_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, PathInData(), &value_values));
+ VariantMap typed_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(typed_json, PathInData(), &typed_values));
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto root_value = value_values.find(PathInData());
+ if (root_value != value_values.end() && !is_empty_object_marker(root_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, PathInData()));
+ value_values.merge(std::move(typed_values));
+ return variant_map_to_json(std::move(value_values), json);
+}
+
+static Status variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ std::string metadata;
+ bool has_metadata = false;
+ RETURN_IF_ERROR(resolve_variant_metadata(variant_field, fields, inherited_metadata, &metadata,
+ &has_metadata));
+
+ std::string typed_json;
+ bool typed_present = false;
+ RETURN_IF_ERROR(variant_typed_value_to_json(variant_field, fields, metadata, &typed_json,
+ &typed_present));
+
+ std::string value_json;
+ bool value_present = false;
+ RETURN_IF_ERROR(variant_residual_value_to_json(variant_field, fields, metadata, has_metadata,
+ &value_json, &value_present));
+
+ if (value_present && typed_present) {
+ RETURN_IF_ERROR(merge_variant_value_and_typed_json(value_json, typed_json, json));
+ *present = true;
+ return Status::OK();
+ }
+
+ if (typed_present) {
+ *json = std::move(typed_json);
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+
+ *present = false;
+ return Status::OK();
+}
+
+static Status shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json, bool* present,
+ bool allow_scalar_typed_value_only_wrapper) {
+ if (is_variant_wrapper_field(field_schema, allow_scalar_typed_value_only_wrapper)) {
+ return variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ if (is_value_only_variant_wrapper_candidate(field_schema)) {
+ Status st = variant_to_json(field_schema, field, &metadata, json, present);
+ if (st.ok()) {
+ return st;
+ }
+ if (!st.is<ErrorCode::CORRUPTION>()) {
+ return st;
+ }
+ }
+ return typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+static Status typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ json->clear();
+ json->push_back('[');
+ for (size_t i = 0; i < elements.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(element_schema, elements[i], metadata, &element_json,
+ &element_present, true));
+ if (!element_present) {
+ return Status::Corruption("Parquet VARIANT array element is missing");
+ }
+ json->append(element_json);
+ }
+ json->push_back(']');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ json->clear();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(typed_value_field.children[i], fields[i], metadata,
+ &child_json, &child_present, false));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ first = false;
+ }
+ json->push_back('}');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ switch (typed_type->get_primitive_type()) {
+ case TYPE_STRUCT:
+ return typed_struct_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_ARRAY:
+ return typed_array_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_MAP: {
+ VariantMap values;
+ PathInDataBuilder path;
+ std::deque<std::string> string_values;
+ RETURN_IF_ERROR(typed_map_to_variant_map(typed_value_field, field, metadata, &path, &values,
+ present, &string_values));
+ if (!*present) {
+ return Status::OK();
+ }
+ return variant_map_to_json(std::move(values), json);
+ }
+ default:
+ return scalar_typed_value_to_json(typed_value_field, field, json, present);
+ }
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values);
+
+static Status variant_to_variant_map(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ VariantMap value_values;
+ bool value_present = false;
+ const PathInData current_path = path->build();
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ RETURN_IF_ERROR(parquet::decode_variant_to_variant_map(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), current_path, &value_values,
+ string_values));
+ }
+ }
+
+ VariantMap typed_values;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_variant_map(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, path,
+ &typed_values, &typed_present, string_values));
+ }
+
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto current_value = value_values.find(current_path);
+ if (value_present && typed_present && current_value != value_values.end() &&
+ !is_empty_object_marker(current_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, current_path));
+ values->merge(std::move(value_values));
+ values->merge(std::move(typed_values));
+ *present = value_present || typed_present;
+ return Status::OK();
+}
+
+static Status shredded_field_to_variant_map(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values) {
+ if (is_variant_wrapper_field(field_schema, false)) {
+ return variant_to_variant_map(field_schema, field, &metadata, path, values, present,
+ string_values);
+ }
+ if (is_value_only_variant_wrapper_candidate(field_schema)) {
+ Status st = variant_to_variant_map(field_schema, field, &metadata, path, values, present,
+ string_values);
+ if (st.ok()) {
+ return st;
+ }
+ if (!st.is<ErrorCode::CORRUPTION>()) {
+ return st;
+ }
+ }
+ return typed_value_to_variant_map(field_schema, field, metadata, path, values, present,
+ string_values);
+}
+
+static Status append_typed_field_to_variant_map(const FieldSchema& typed_value_field,
+ const Field& field, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static void move_variant_map_to_field(VariantMap&& element_values, FieldWithDataType* value) {
+ if (element_values.size() == 1 && element_values.begin()->first.empty()) {
+ *value = std::move(element_values.begin()->second);
+ return;
+ }
+ value->field = Field::create_field<TYPE_VARIANT>(std::move(element_values));
+ fill_variant_field_info(value);
+}
+
+static Status typed_array_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present,
+ std::deque<std::string>* string_values) {
+ if ((contains_uuid_typed_value_field(typed_value_field) ||
+ contains_temporal_variant_leaf_type(typed_value_field.data_type) ||
+ contains_floating_point_variant_leaf_type(typed_value_field.data_type)) &&
+ is_direct_variant_leaf_type(typed_value_field.data_type)) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(fill_direct_array_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+ }
+ if (is_direct_variant_leaf_type(typed_value_field.data_type)) {
+ return append_typed_field_to_variant_map(typed_value_field, field, path, values, present);
+ }
+
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ Array array;
+ array.reserve(elements.size());
+ for (const auto& element : elements) {
+ VariantMap element_values;
+ bool element_present = false;
+ PathInDataBuilder element_path;
+ RETURN_IF_ERROR(shredded_field_to_variant_map(element_schema, element, metadata,
+ &element_path, &element_values,
+ &element_present, string_values));
Review Comment:
This treats a null complex element in a typed VARIANT array as file corruption. For a valid value like `[null, {"a":1}]` shredded as `typed_value: list<optional struct<a:int>>`, the null element reaches `shredded_field_to_variant_map()` with `present=false` and fails here instead of preserving a null array element. The map path below already materializes absent values as `Field()`, so arrays need equivalent null-element handling rather than rejecting the row. Please add coverage for typed VARIANT arrays containing null complex elements.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469499622
<details>
<summary>TPC-DS: <b>Total hot run time: 169362 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 87a7378a43c24b89233a6fb138ed1b79d6776812, data reload: false
query5 4316 658 535 535
query6 368 224 214 214
query7 4382 569 302 302
query8 324 227 209 209
query9 8819 3986 3997 3986
query10 458 345 296 296
query11 5800 2325 2204 2204
query12 192 126 125 125
query13 1330 628 456 456
query14 6001 5348 5154 5154
query14_1 4490 4460 4489 4460
query15 213 209 187 187
query16 993 477 460 460
query17 1154 753 607 607
query18 2505 507 379 379
query19 227 212 170 170
query20 140 132 136 132
query21 215 140 120 120
query22 13612 13541 13340 13340
query23 17287 16550 16039 16039
query23_1 16202 16196 16206 16196
query24 7498 1771 1287 1287
query24_1 1320 1293 1329 1293
query25 609 505 457 457
query26 1323 320 176 176
query27 2676 584 341 341
query28 4469 1958 1932 1932
query29 1013 659 509 509
query30 303 241 200 200
query31 1141 1084 954 954
query32 86 81 76 76
query33 544 366 317 317
query34 1159 1115 633 633
query35 763 775 684 684
query36 1326 1378 1184 1184
query37 149 102 99 99
query38 3199 3154 3043 3043
query39 923 943 890 890
query39_1 865 888 868 868
query40 231 146 122 122
query41 65 68 64 64
query42 111 105 107 105
query43 324 319 283 283
query44
query45 203 202 192 192
query46 1061 1227 703 703
query47 2317 2344 2212 2212
query48 381 409 284 284
query49 627 489 385 385
query50 971 353 253 253
query51 4268 4337 4208 4208
query52 105 105 98 98
query53 255 285 198 198
query54 308 259 251 251
query55 93 90 84 84
query56 296 299 303 299
query57 1418 1387 1312 1312
query58 296 270 266 266
query59 1531 1647 1396 1396
query60 319 319 313 313
query61 165 163 160 160
query62 663 618 548 548
query63 239 192 201 192
query64 2420 823 662 662
query65
query66 1750 476 358 358
query67 30034 29946 29179 29179
query68
query69 476 341 348 341
query70 1045 1003 958 958
query71 305 273 274 273
query72 2958 2689 2444 2444
query73 870 787 418 418
query74 5051 4902 4747 4747
query75 2661 2619 2249 2249
query76 2301 1157 771 771
query77 390 398 333 333
query78 12184 12201 11637 11637
query79 1467 1037 741 741
query80 658 555 456 456
query81 454 275 247 247
query82 1351 157 125 125
query83 352 278 254 254
query84 259 140 117 117
query85 881 549 460 460
query86 381 345 310 310
query87 3422 3385 3218 3218
query88 3495 2640 2625 2625
query89 441 393 338 338
query90 2040 179 181 179
query91 179 168 140 140
query92 78 79 72 72
query93 1470 1427 844 844
query94 541 361 321 321
query95 678 375 425 375
query96 1017 738 353 353
query97 2690 2691 2577 2577
query98 243 230 223 223
query99 1107 1118 973 973
Total cold run time: 253323 ms
Total hot run time: 169362 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469457200
# FE UT Coverage Report
Increment line coverage ` 76.38% (207/271)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_8587400dc65063b8e8ea52ca286f3e7831788e4a/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_8587400dc65063b8e8ea52ca286f3e7831788e4a/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467114189
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467385329
<details>
<summary>TPC-DS: <b>Total hot run time: 170681 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 8468e1e4b59fc6dd19d32fdad2b2fbf0fbc402c4, data reload: false
query5 4328 650 519 519
query6 336 213 211 211
query7 4401 569 304 304
query8 337 230 218 218
query9 8803 3993 4033 3993
query10 438 340 292 292
query11 5787 2395 2235 2235
query12 181 135 125 125
query13 1281 635 443 443
query14 5957 5381 5056 5056
query14_1 4369 4384 4348 4348
query15 211 205 185 185
query16 1032 448 462 448
query17 1144 750 611 611
query18 2604 502 365 365
query19 220 206 166 166
query20 138 133 129 129
query21 217 144 119 119
query22 13602 13584 13397 13397
query23 17276 16342 16035 16035
query23_1 16154 16203 16196 16196
query24 7397 1747 1309 1309
query24_1 1321 1319 1308 1308
query25 593 501 437 437
query26 1338 335 180 180
query27 2704 546 345 345
query28 4420 2001 1938 1938
query29 986 642 520 520
query30 303 243 197 197
query31 1112 1070 947 947
query32 94 81 75 75
query33 554 348 342 342
query34 1144 1131 648 648
query35 760 802 670 670
query36 1340 1376 1138 1138
query37 151 114 100 100
query38 3224 3148 3060 3060
query39 936 908 899 899
query39_1 886 860 893 860
query40 226 145 124 124
query41 65 65 62 62
query42 108 106 111 106
query43 327 326 280 280
query44
query45 211 200 193 193
query46 1045 1202 739 739
query47 2358 2297 2208 2208
query48 393 420 287 287
query49 627 484 377 377
query50 971 357 262 262
query51 4277 4233 4214 4214
query52 104 104 94 94
query53 256 282 203 203
query54 330 269 254 254
query55 94 88 86 86
query56 299 313 298 298
query57 1395 1364 1309 1309
query58 297 273 267 267
query59 1601 1636 1422 1422
query60 320 324 307 307
query61 164 157 162 157
query62 676 619 558 558
query63 244 205 202 202
query64 2386 815 640 640
query65
query66 1678 474 351 351
query67 30086 29924 29859 29859
query68
query69 466 364 310 310
query70 1027 1007 1047 1007
query71 310 271 269 269
query72 3036 2755 2443 2443
query73 859 747 445 445
query74 5065 4944 4721 4721
query75 2659 2585 2282 2282
query76 2332 1108 752 752
query77 416 400 332 332
query78 12182 12231 11550 11550
query79 1413 1054 735 735
query80 638 549 450 450
query81 456 294 244 244
query82 1384 156 121 121
query83 355 280 248 248
query84 265 148 117 117
query85 882 531 453 453
query86 390 343 340 340
query87 3458 3345 3221 3221
query88 3530 2666 2645 2645
query89 440 387 332 332
query90 1992 185 190 185
query91 179 164 151 151
query92 78 79 69 69
query93 1556 1435 868 868
query94 525 363 301 301
query95 684 385 347 347
query96 1045 787 332 332
query97 2703 2694 2558 2558
query98 236 227 231 227
query99 1153 1145 1034 1034
Total cold run time: 253362 ms
Total hot run time: 170681 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467681481
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468991829
run p0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468991575
run p0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469136297
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469195508
# FE UT Coverage Report
Increment line coverage ` 78.22% (158/202)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_6b22fa723d69d717244b1cd0c3894901ce9a5b9b/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_6b22fa723d69d717244b1cd0c3894901ce9a5b9b/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469247626
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466963870
<details>
<summary>TPC-H: <b>Total hot run time: 31004 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 67c53d3c346f8064df2eeeee8b5cd103c13cdd4f, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17626 3872 3930 3872
q2 q3 10796 1420 820 820
q4 4701 475 353 353
q5 7819 2218 2034 2034
q6 370 177 140 140
q7 962 782 622 622
q8 9348 1708 1656 1656
q9 6967 4882 4872 4872
q10 6451 2107 1792 1792
q11 443 267 233 233
q12 700 428 288 288
q13 18238 3356 2770 2770
q14 264 261 226 226
q15 q16 811 778 702 702
q17 1027 978 945 945
q18 6945 5830 5507 5507
q19 1611 1295 1095 1095
q20 516 396 260 260
q21 6063 2639 2511 2511
q22 452 361 306 306
Total cold run time: 102110 ms
Total hot run time: 31004 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4200 4102 4090 4090
q2 q3 4474 4888 4306 4306
q4 2094 2187 1378 1378
q5 4358 4302 4515 4302
q6 258 194 139 139
q7 1902 1870 1667 1667
q8 2475 2070 2067 2067
q9 7702 7738 7654 7654
q10 4514 4431 3989 3989
q11 775 417 381 381
q12 727 764 506 506
q13 3259 3659 2977 2977
q14 306 301 285 285
q15 q16 737 752 635 635
q17 1316 1328 1309 1309
q18 7743 7423 6953 6953
q19 1158 1154 1144 1144
q20 2256 2237 1942 1942
q21 5370 4733 4508 4508
q22 553 463 432 432
Total cold run time: 56177 ms
Total hot run time: 50664 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468736448
# FE UT Coverage Report
Increment line coverage ` 76.76% (142/185)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_1ca8493b6b6868ea7b8ca7c5ea4b6a4848b4e1c7/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_1ca8493b6b6868ea7b8ca7c5ea4b6a4848b4e1c7/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3254240654
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java:
##########
@@ -462,11 +462,25 @@ private LogicalPlan buildMergePlan(ConnectContext ctx, IcebergExternalTable iceb
icebergTable.getBaseSchema(true),
outputExprs,
deleteCtx,
+ writesDataFiles(matchedClauses, notMatchedClauses),
Optional.empty(),
Review Comment:
Passing `writesDataFiles=false` only disables the sink-side VARIANT write validation; it does not stop the delete-only plan from reading every visible target column. `buildDeleteProjection()` still appends all visible columns, so a delete-only MERGE whose join/predicate uses only non-VARIANT columns will still materialize a VARIANT column. On an ORC Iceberg table this reaches `IcebergScanNode.validateVariantReadSupported()` and fails with the Parquet-only VARIANT read error even though the delete only needs row-id/delete metadata, and on Parquet it performs unnecessary VARIANT reads. This is distinct from the earlier write-validation thread: the remaining failure is introduced by the read/projection side after the new flag is passed. Please make delete-only MERGE project only the operation, row id, lineage/delete columns, and columns required by the join/predicate, or otherwise avoid scanning unused VARIANT columns; add coverage for delete-only MERGE on a table with an unused VARI
ANT column, including a non-Parquet table if possible.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java:
##########
@@ -265,39 +299,86 @@ public Void visitAlias(Alias alias, CollectorContext context) {
@Override
public Void visitCast(Cast cast, CollectorContext context) {
+ Expression child = cast.child(0);
+ if (child.getDataType() instanceof VariantType && context.accessPathBuilder.isEmpty()) {
+ CollectorContext variantRootContext = context.copy();
+ variantRootContext.setCollectVariantRoot(true);
+ return continueCollectAccessPath(child, variantRootContext);
+ }
if (!context.accessPathBuilder.isEmpty()
&& cast.getDataType() instanceof NestedColumnPrunable
Review Comment:
This cast handling still drops a non-empty VARIANT access suffix whenever the cast child is `VariantType`, because `VariantType` is not `NestedColumnPrunable` and the method falls through to visiting the child with a fresh empty context. For example, `SELECT cast(v AS variant)['a'] FROM variant_tbl WHERE v['k'] IS NOT NULL` reaches `visitCast()` with suffix `a`; the suffix is discarded here, the sibling predicate can leave only `[v, k]` at the scan, and the projection then evaluates `['a']` against a pruned VARIANT. The same pattern applies to casts from a VARIANT subpath into complex types, e.g. selecting a field from `cast(v['obj'] AS struct<...>)`. Please propagate the access builder through VARIANT-preserving / VARIANT-to-complex casts where the suffix still refers to the child value, and add coverage combining such a casted subpath projection with a sibling VARIANT predicate.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470559833
# FE Regression Coverage Report
Increment line coverage ` 43.32% (146/337)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_b7402399a565dc0a3e621ecb4d61cfd48a4eb2fc_merge_fe/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_b7402399a565dc0a3e621ecb4d61cfd48a4eb2fc_merge_fe/report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253073742
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,949 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <deque>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+#include "core/column/column_variant.h"
+#include "core/value/jsonb_value.h"
+#include "exec/common/variant_util.h"
+
+namespace doris::parquet {
+
+std::string format_variant_uuid(const uint8_t* ptr) {
+ static constexpr char hex[] = "0123456789abcdef";
+ std::string uuid;
+ uuid.reserve(36);
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ uuid.push_back('-');
+ }
+ uuid.push_back(hex[ptr[i] >> 4]);
+ uuid.push_back(hex[ptr[i] & 0x0f]);
+ }
+ return uuid;
+}
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+struct VariantObjectLayout {
+ std::vector<uint64_t> field_ids;
+ std::vector<uint64_t> field_offsets;
+ const uint8_t* fields = nullptr;
+ uint64_t total_size = 0;
+};
+
+struct VariantArrayLayout {
+ std::vector<uint64_t> field_offsets;
+ const uint8_t* fields = nullptr;
+ uint64_t total_size = 0;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+__int128 read_signed_int128_le(const uint8_t* ptr) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ static constexpr unsigned __int128 sign_bit = static_cast<unsigned __int128>(1) << 127;
+ if ((unsigned_value & sign_bit) == 0) {
+ return static_cast<__int128>(unsigned_value);
+ }
+ static constexpr __int128 signed_half_range = static_cast<__int128>(1) << 126;
+ return (static_cast<__int128>(unsigned_value & (sign_bit - 1)) - signed_half_range) -
+ signed_half_range;
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ if (size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+bool variant_string_less(std::string_view lhs, std::string_view rhs) {
+ return std::lexicographical_compare(
+ lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](char left, char right) {
+ return static_cast<unsigned char>(left) < static_cast<unsigned char>(right);
+ });
+}
+
+bool is_valid_utf8(std::string_view value) {
+ const auto* data = reinterpret_cast<const uint8_t*>(value.data());
+ const auto* end = data + value.size();
+ while (data < end) {
+ const uint8_t first = *data++;
+ if (first <= 0x7f) {
+ continue;
+ }
+
+ uint32_t code_point = 0;
+ size_t continuation_bytes = 0;
+ if (first >= 0xc2 && first <= 0xdf) {
+ code_point = first & 0x1f;
+ continuation_bytes = 1;
+ } else if (first >= 0xe0 && first <= 0xef) {
+ code_point = first & 0x0f;
+ continuation_bytes = 2;
+ } else if (first >= 0xf0 && first <= 0xf4) {
+ code_point = first & 0x07;
+ continuation_bytes = 3;
+ } else {
+ return false;
+ }
+
+ if (static_cast<size_t>(end - data) < continuation_bytes) {
+ return false;
+ }
+ for (size_t i = 0; i < continuation_bytes; ++i) {
+ const uint8_t byte = *data++;
+ if ((byte & 0xc0) != 0x80) {
+ return false;
+ }
+ code_point = (code_point << 6) | (byte & 0x3f);
+ }
+
+ if ((continuation_bytes == 2 && code_point < 0x800) ||
+ (continuation_bytes == 3 && code_point < 0x10000) ||
+ (code_point >= 0xd800 && code_point <= 0xdfff) || code_point > 0x10ffff) {
+ return false;
+ }
+ }
+ return true;
+}
+
+Status require_valid_utf8(std::string_view value, std::string_view context) {
+ if (!is_valid_utf8(value)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} UTF-8 string", context);
+ }
+ return Status::OK();
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+Status append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point value is not supported");
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+ return Status::OK();
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next);
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ json->push_back('"');
+ json->append(format_variant_uuid(ptr));
+ json->push_back('"');
+}
+
+Status make_jsonb_field(std::string_view json, FieldWithDataType* value) {
+ JsonBinaryValue jsonb_value;
+ RETURN_IF_ERROR(jsonb_value.from_json_string(json.data(), json.size()));
+ value->field =
+ Field::create_field<TYPE_JSONB>(JsonbField(jsonb_value.value(), jsonb_value.size()));
+ value->base_scalar_type_id = TYPE_JSONB;
+ return Status::OK();
+}
+
+Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field("{}", &value));
+ (*values)[path] = std::move(value);
+ return Status::OK();
+}
+
+Status parse_json_to_variant_map(std::string_view json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (parsed.is_null()) {
+ return Status::OK();
+ }
+
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ for (auto& [parsed_path, value] : parsed.get<TYPE_VARIANT>()) {
+ path.append(parsed_path.get_parts(), false);
+ (*values)[path.build()] = std::move(value);
+ for (size_t i = 0; i < parsed_path.get_parts().size(); ++i) {
+ path.pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+void fill_field_type_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+ value->precision = info.precision;
+ value->scale = info.scale;
+}
+
+Status read_array_layout(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ VariantArrayLayout* layout) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int num_elements_size = (value_header & 0x04) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "array element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "array field offsets"));
+ layout->field_offsets.resize(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ layout->field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ layout->total_size = layout->field_offsets.back();
+ layout->fields = ptr;
+ RETURN_IF_ERROR(
+ require_available(layout->fields, end, layout->total_size, "array field values"));
+ RETURN_IF_ERROR(
+ validate_array_field_offsets(layout->field_offsets, layout->total_size, "array"));
+ return Status::OK();
+}
+
+Status read_object_layout(uint8_t value_header, const uint8_t* ptr, const uint8_t* end,
+ const VariantMetadata& metadata, VariantObjectLayout* layout) {
+ int field_offset_size = (value_header & 0x03) + 1;
+ int field_id_size = ((value_header >> 2) & 0x03) + 1;
+ int num_elements_size = (value_header & 0x10) != 0 ? 4 : 1;
+
+ RETURN_IF_ERROR(require_available(ptr, end, num_elements_size, "object element count"));
+ uint64_t num_elements = read_unsigned_le(ptr, num_elements_size);
+ ptr += num_elements_size;
+
+ RETURN_IF_ERROR(
+ require_available_entries(ptr, end, num_elements, field_id_size, "object field ids"));
+ layout->field_ids.resize(num_elements);
+ for (uint64_t i = 0; i < num_elements; ++i) {
+ layout->field_ids[i] = read_unsigned_le(ptr, field_id_size);
+ ptr += field_id_size;
+ if (layout->field_ids[i] >= metadata.dictionary.size()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field id {}",
+ layout->field_ids[i]);
+ }
+ if (i > 0 && !variant_string_less(metadata.dictionary[layout->field_ids[i - 1]],
+ metadata.dictionary[layout->field_ids[i]])) {
+ return Status::Corruption("Invalid Parquet VARIANT object field names");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, num_elements + 1, field_offset_size,
+ "object field offsets"));
+ layout->field_offsets.resize(num_elements + 1);
+ for (uint64_t i = 0; i <= num_elements; ++i) {
+ layout->field_offsets[i] = read_unsigned_le(ptr, field_offset_size);
+ ptr += field_offset_size;
+ }
+
+ layout->total_size = layout->field_offsets.back();
+ layout->fields = ptr;
+ RETURN_IF_ERROR(
+ require_available(layout->fields, end, layout->total_size, "object field values"));
+ RETURN_IF_ERROR(
+ validate_array_field_offsets(layout->field_offsets, layout->total_size, "object"));
Review Comment:
The VariantMap decoder rejects valid VARIANT objects whose field values are stored out of field-name order. The Parquet VARIANT spec requires field ids/offset entries to be listed in key order, but explicitly says the actual value entries may be in any physical order, so object offsets need not be monotonically increasing. The JSON decoder below handles this with `compute_object_field_ends()`, but this shared layout reader calls `validate_array_field_offsets()` and `decode_object_to_variant_map()` later slices children with `field_offsets[i + 1]`. A valid residual object with keys `[a,b]` but physical values stored as `b` then `a` will fail here or be mis-sliced on the `VariantMap` path used by `variant_to_variant_map()`, even though `decode_variant_to_json()` can read it. Please compute per-field ends for object layouts instead of requiring monotonic offsets, and add coverage for `decode_variant_to_variant_map()` with non-monotonic object offsets.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468190692
<details>
<summary>TPC-DS: <b>Total hot run time: 170726 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 788ad7a017b7fb12d358bc3f6112c03aea0d3be6, data reload: false
query5 4317 678 516 516
query6 425 244 199 199
query7 4343 587 305 305
query8 326 233 221 221
query9 8833 4072 4065 4065
query10 474 352 313 313
query11 5806 2393 2213 2213
query12 181 133 129 129
query13 1291 637 413 413
query14 6261 5548 5238 5238
query14_1 4530 4544 4501 4501
query15 223 213 186 186
query16 1030 464 450 450
query17 1155 770 623 623
query18 2736 533 365 365
query19 237 210 177 177
query20 143 135 135 135
query21 232 144 124 124
query22 13617 13600 13341 13341
query23 17371 16373 15921 15921
query23_1 16157 16133 16158 16133
query24 7625 1788 1297 1297
query24_1 1323 1329 1311 1311
query25 553 499 415 415
query26 1306 349 172 172
query27 2664 561 336 336
query28 4478 1968 1982 1968
query29 985 642 487 487
query30 309 244 203 203
query31 1155 1078 950 950
query32 87 77 74 74
query33 547 370 298 298
query34 1182 1146 657 657
query35 805 800 710 710
query36 1307 1318 1177 1177
query37 161 105 95 95
query38 3211 3148 3085 3085
query39 918 926 884 884
query39_1 870 876 879 876
query40 231 152 126 126
query41 68 66 62 62
query42 115 112 110 110
query43 336 343 299 299
query44
query45 211 198 195 195
query46 1088 1188 756 756
query47 2228 2299 2159 2159
query48 400 426 291 291
query49 647 521 395 395
query50 1053 369 258 258
query51 4313 4244 4246 4244
query52 105 111 94 94
query53 257 290 207 207
query54 330 280 256 256
query55 93 93 88 88
query56 314 323 306 306
query57 1394 1377 1268 1268
query58 333 265 272 265
query59 1544 1631 1403 1403
query60 335 330 312 312
query61 160 156 162 156
query62 667 634 551 551
query63 252 202 203 202
query64 2401 816 648 648
query65
query66 1657 490 370 370
query67 29986 29982 29804 29804
query68
query69 486 342 303 303
query70 1039 1079 1025 1025
query71 316 288 279 279
query72 3027 2781 2430 2430
query73 835 781 433 433
query74 5086 4950 4768 4768
query75 2833 2733 2302 2302
query76 2311 1218 815 815
query77 407 430 341 341
query78 12246 12228 11616 11616
query79 1286 1032 769 769
query80 617 571 460 460
query81 449 288 239 239
query82 247 162 131 131
query83 269 280 248 248
query84 267 143 111 111
query85 884 538 459 459
query86 375 332 329 329
query87 3438 3364 3224 3224
query88 3614 2667 2637 2637
query89 439 386 337 337
query90 2184 188 188 188
query91 193 168 141 141
query92 80 79 74 74
query93 1476 1434 844 844
query94 519 359 318 318
query95 681 495 346 346
query96 1059 815 322 322
query97 2682 2669 2541 2541
query98 235 235 232 232
query99 1132 1097 963 963
Total cold run time: 253337 ms
Total hot run time: 170726 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468307847
# FE UT Coverage Report
Increment line coverage ` 64.91% (74/114)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_bbbd592037e2e049054fb61809afba78dfbdf627/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_bbbd592037e2e049054fb61809afba78dfbdf627/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468387499
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468606820
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468606759
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253879207
##########
be/src/format/parquet/parquet_nested_column_utils.cpp:
##########
@@ -0,0 +1,555 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <cctype>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "core/data_type/data_type_nullable.h"
+#include "format/parquet/schema_desc.h"
+
+namespace doris {
+namespace {
+
+enum class NestedPathMode {
+ NAME,
+ FIELD_ID,
+};
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_variant_typed_child_by_key(const FieldSchema& field_schema,
+ std::string_view key, NestedPathMode mode) {
+ if (const auto* child = find_child_by_exact_name(field_schema, key)) {
+ return child;
+ }
+ if (mode == NestedPathMode::NAME) {
+ return nullptr;
+ }
+ for (const auto& child : field_schema.children) {
Review Comment:
In the Iceberg path this helper runs in `FIELD_ID` mode, but VARIANT subpaths are still logical VARIANT object keys, not Iceberg field IDs: `SlotTypeReplacer` leaves `VariantType` path components unchanged after converting only the top-level Iceberg column id. With this fallback, a query for a numeric key that is absent, for example `v['20']`, can match a different shredded typed child whose Parquet/Iceberg field id is `20` and select that column instead of falling back to the residual/missing-key behavior. That can return the wrong sibling or suppress the residual `value` path. This is distinct from the existing map-key field-id serialization thread because it is the BE VARIANT typed-child lookup interpreting user keys as field ids. Please keep VARIANT keys name-based in field-id mode (or only use field-id matching for paths that FE actually serialized as field ids) and add coverage for numeric-looking VARIANT keys colliding with another typed child's field id.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468817689
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3252894731
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,576 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+__int128 read_signed_int128_le(const uint8_t* ptr) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ static constexpr unsigned __int128 sign_bit = static_cast<unsigned __int128>(1) << 127;
+ if ((unsigned_value & sign_bit) == 0) {
+ return static_cast<__int128>(unsigned_value);
+ }
+ static constexpr __int128 signed_half_range = static_cast<__int128>(1) << 126;
+ return (static_cast<__int128>(unsigned_value & (sign_bit - 1)) - signed_half_range) -
+ signed_half_range;
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+bool variant_string_less(std::string_view lhs, std::string_view rhs) {
+ return std::lexicographical_compare(
+ lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](char left, char right) {
+ return static_cast<unsigned char>(left) < static_cast<unsigned char>(right);
+ });
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+Status append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point value is not supported");
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+ return Status::OK();
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
Review Comment:
The metadata header accepts reserved bit 5 silently. The current grammar only defines the low 4 version bits, bit 4 for `sorted_strings`, and bits 6-7 for offset width, so a header such as `0x21` is not a valid version-1 encoding but is decoded as normal metadata. Please reject headers with reserved bits set and add a malformed-metadata unit case.
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,576 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+__int128 read_signed_int128_le(const uint8_t* ptr) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ static constexpr unsigned __int128 sign_bit = static_cast<unsigned __int128>(1) << 127;
+ if ((unsigned_value & sign_bit) == 0) {
+ return static_cast<__int128>(unsigned_value);
+ }
+ static constexpr __int128 signed_half_range = static_cast<__int128>(1) << 126;
+ return (static_cast<__int128>(unsigned_value & (sign_bit - 1)) - signed_half_range) -
+ signed_half_range;
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+bool variant_string_less(std::string_view lhs, std::string_view rhs) {
+ return std::lexicographical_compare(
+ lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](char left, char right) {
+ return static_cast<unsigned char>(left) < static_cast<unsigned char>(right);
+ });
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
Review Comment:
`append_json_string()` escapes control characters but otherwise copies arbitrary bytes into the JSON output. Parquet VARIANT metadata keys and string values are UTF-8 strings, so a corrupt payload containing bytes like `0xff` in a dictionary key or short-string value is currently accepted and returned as invalid JSON bytes instead of corruption. Please validate UTF-8 before appending these strings (or reject invalid metadata/string payloads at decode time) and add malformed VARIANT coverage.
##########
be/src/format/parquet/vparquet_reader.cpp:
##########
@@ -432,11 +440,111 @@ Status ParquetReader::on_before_init_reader(ReaderInitContext* ctx) {
RETURN_IF_ERROR(get_file_metadata_schema(&field_desc));
RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
ctx->tuple_descriptor, *field_desc, ctx->table_info_node));
+ auto column_id_result = _create_column_ids_by_name(field_desc, ctx->tuple_descriptor);
+ ctx->column_ids = std::move(column_id_result.column_ids);
+ ctx->filter_column_ids = std::move(column_id_result.filter_column_ids);
}
return Status::OK();
}
+ColumnIdResult ParquetReader::_create_column_ids_by_name(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<std::string, const FieldSchema*> table_col_name_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) {
+ continue;
+ }
+ table_col_name_to_field_schema_map[field_schema->lower_case_name] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ process_nested_access_paths(
+ parquet_field, access_paths, out_ids,
+ [](const FieldSchema* field) { return field->get_column_id(); },
+ [](const FieldSchema* field) { return field->get_max_column_id(); },
+ HiveParquetNestedColumnUtils::extract_nested_column_ids);
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_name_to_field_schema_map.find(slot->col_name_lower_case());
+ if (it == table_col_name_to_field_schema_map.end()) {
+ continue;
+ }
+ auto field_schema = it->second;
+
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP && slot->col_type() != TYPE_VARIANT)) {
+ column_ids.insert(field_schema->column_id);
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
Review Comment:
Full VARIANT projection is still pruned when the same slot also has a nested access path. The full-root fallback in `process_nested_access_paths()` only fires when `all_access_paths()` is empty or contains a top-level-only path, but FE does not add a root path for a plain projection of `v`; if a predicate or expression also references `v['metric']`, this vector contains only the nested path and the reader selects just that shard. A query like `SELECT cast(v AS string) FROM local(...) WHERE cast(v['metric'] AS bigint) > 0` can therefore reconstruct a partial VARIANT with sibling fields pruned. The same pattern is present in the Hive and Iceberg reader column-id builders. Please ensure a slot that needs the full column keeps the full field range even when nested predicate/access paths are also present, and add coverage for full `v` projection plus a nested predicate/subpath.
##########
be/src/format/parquet/schema_desc.cpp:
##########
@@ -446,6 +463,39 @@ Status FieldDescriptor::parse_group_field(const std::vector<tparquet::SchemaElem
return Status::OK();
}
+Status FieldDescriptor::parse_variant_field(const std::vector<tparquet::SchemaElement>& t_schemas,
+ size_t curr_pos, FieldSchema* variant_field) {
+ RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, variant_field));
+
Review Comment:
`parse_variant_field()` accepts extra children under a `VARIANT` annotated group and the reader later ignores them. A malformed schema like `VARIANT { required binary metadata; optional binary value; optional int32 extra; }` passes this validation, but `variant_to_variant_map()` only consumes `metadata`, `value`, and `typed_value`, silently dropping `extra`. Please reject unrecognized top-level VARIANT children so malformed files fail instead of losing data.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +123,1126 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper,
+ bool allow_value_only_wrapper = false) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr || allow_value_only_wrapper;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ JsonBinaryValue empty_object;
+ RETURN_IF_ERROR(empty_object.from_json_string("{}"));
+ (*values)[path] = FieldWithDataType {.field = Field::create_field<TYPE_JSONB>(JsonbField(
+ empty_object.value(), empty_object.size())),
+ .base_scalar_type_id = TYPE_JSONB};
+ return Status::OK();
+}
+
+static bool is_empty_object_marker(const FieldWithDataType& value) {
+ if (value.field.get_type() != TYPE_JSONB) {
+ return false;
+ }
+ const auto& jsonb = value.field.get<TYPE_JSONB>();
+ const JsonbDocument* document = nullptr;
+ Status st =
+ JsonbDocument::checkAndCreateDocument(jsonb.get_value(), jsonb.get_size(), &document);
+ if (!st.ok() || document == nullptr || document->getValue() == nullptr ||
+ !document->getValue()->isObject()) {
+ return false;
+ }
+ return document->getValue()->unpack<ObjectVal>()->numElem() == 0;
+}
+
+static Status collect_empty_object_markers(const rapidjson::Value& value, PathInDataBuilder* path,
+ VariantMap* values) {
+ if (!value.IsObject()) {
+ return Status::OK();
+ }
+ if (value.MemberCount() == 0) {
+ return insert_empty_object_marker(path->build(), values);
+ }
+ for (auto it = value.MemberBegin(); it != value.MemberEnd(); ++it) {
+ if (it->value.IsObject()) {
+ path->append(std::string_view(it->name.GetString(), it->name.GetStringLength()), false);
+ RETURN_IF_ERROR(collect_empty_object_markers(it->value, path, values));
+ path->pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+static Status add_empty_object_markers_from_json(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ if (json.find("{}") == std::string::npos) {
+ return Status::OK();
+ }
+ rapidjson::Document document;
+ document.Parse(json.data(), json.size());
+ if (document.HasParseError()) {
+ return Status::Corruption("Invalid Parquet VARIANT decoded JSON");
+ }
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ return collect_empty_object_markers(document, &path, values);
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (!parsed.is_null()) {
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ }
+ RETURN_IF_ERROR(add_empty_object_markers_from_json(json, prefix, values));
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static bool path_has_prefix(const PathInData& path, const PathInData& prefix) {
+ const auto& parts = path.get_parts();
+ const auto& prefix_parts = prefix.get_parts();
+ if (parts.size() < prefix_parts.size()) {
+ return false;
+ }
+ for (size_t i = 0; i < prefix_parts.size(); ++i) {
+ if (parts[i] != prefix_parts[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool has_descendant_path(const VariantMap& values, const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ return std::ranges::any_of(values, [&](const auto& entry) {
+ const auto& path = entry.first;
+ return path.get_parts().size() > prefix_size && path_has_prefix(path, prefix);
+ });
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* values,
+ const VariantMap& shadowing_values) {
+ for (auto it = values->begin(); it != values->end();) {
+ if (is_empty_object_marker(it->second) &&
+ (has_descendant_path(*values, it->first) ||
+ has_descendant_path(shadowing_values, it->first))) {
+ it = values->erase(it);
+ continue;
+ }
+ ++it;
+ }
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* value_values,
+ VariantMap* typed_values) {
+ erase_shadowed_empty_object_markers(value_values, *typed_values);
+ erase_shadowed_empty_object_markers(typed_values, *value_values);
+}
+
+static Status check_no_shredded_value_typed_duplicates(const VariantMap& value_values,
+ const VariantMap& typed_values,
+ const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ for (const auto& value_entry : value_values) {
+ const auto& value_path = value_entry.first;
+ if (!path_has_prefix(value_path, prefix)) {
+ continue;
+ }
+ if (value_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(value_entry.second) &&
+ !has_descendant_path(typed_values, value_path)) {
+ continue;
+ }
+ if (!typed_values.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT residual value conflicts with typed_value at path {}",
+ value_path.get_path());
+ }
+ continue;
+ }
+ for (const auto& typed_entry : typed_values) {
+ const auto& typed_path = typed_entry.first;
+ if (!path_has_prefix(typed_path, prefix)) {
+ continue;
+ }
+ if (typed_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(typed_entry.second) &&
+ !has_descendant_path(value_values, typed_path)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ if (value_path.get_parts()[prefix_size] == typed_path.get_parts()[prefix_size]) {
+ if (value_path == typed_path && is_empty_object_marker(value_entry.second) &&
+ is_empty_object_marker(typed_entry.second)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+static bool has_direct_typed_parent_null(const std::vector<const NullMap*>& null_maps, size_t row) {
+ return std::ranges::any_of(null_maps, [&](const NullMap* null_map) {
+ DCHECK_LT(row, null_map->size());
+ return (*null_map)[row];
+ });
+}
+
+static void insert_direct_typed_leaf_range(const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps,
+ IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ nullable_leaf.get_nested_column().insert_range_from(*value_column, start, rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static bool is_temporal_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static DataTypePtr direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return std::make_shared<DataTypeInt64>();
+ }
+ return type;
+}
+
+static bool contains_temporal_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_temporal_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static int64_t direct_temporal_variant_value(PrimitiveType type, const IColumn& column,
+ size_t row) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ return static_cast<int64_t>(
+ std::llround(assert_cast<const ColumnTimeV2&>(column).get_data()[row]));
+ case TYPE_DATE:
+ return variant_date_value(assert_cast<const ColumnDate&>(column).get_data()[row]);
+ case TYPE_DATETIME:
+ return variant_datetime_value(assert_cast<const ColumnDateTime&>(column).get_data()[row]);
+ case TYPE_DATEV2:
+ return variant_date_value(assert_cast<const ColumnDateV2&>(column).get_data()[row]);
+ case TYPE_DATETIMEV2:
+ return variant_datetime_value(assert_cast<const ColumnDateTimeV2&>(column).get_data()[row]);
+ case TYPE_TIMESTAMPTZ:
+ return variant_datetime_value(
+ assert_cast<const ColumnTimeStampTz&>(column).get_data()[row]);
+ default:
+ DORIS_CHECK(false);
+ return 0;
+ }
+}
+
+static void insert_direct_typed_temporal_leaf_range(
+ PrimitiveType type, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnInt64&>(nullable_leaf.get_nested_column()).get_data();
+ data.reserve(data.size() + rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ data.push_back(direct_temporal_variant_value(type, *value_column, row));
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ return true;
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return !contains_temporal_variant_leaf_type(array_type->get_nested_type()) &&
+ is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool has_selected_direct_typed_leaf(const FieldSchema& field_schema,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return false;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ return has_selected_direct_typed_leaf(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_direct_typed_leaf(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool can_use_direct_typed_only_value(const FieldSchema& variant_field,
+ const std::set<uint64_t>& column_ids) {
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ return (value_idx < 0 || !has_selected_column(variant_field.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ has_selected_direct_typed_leaf(variant_field.children[typed_value_idx], false,
+ column_ids) &&
+ can_direct_read_typed_value(variant_field.children[typed_value_idx], false, column_ids);
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static void fill_variant_leaf_type_info(const DataTypePtr& data_type, FieldWithDataType* value) {
+ auto leaf_type = remove_nullable(data_type);
+ while (leaf_type->get_primitive_type() == TYPE_ARRAY) {
+ leaf_type = remove_nullable(
+ assert_cast<const DataTypeArray*>(leaf_type.get())->get_nested_type());
+ }
+ if (is_decimal(leaf_type->get_primitive_type())) {
+ value->precision = leaf_type->get_precision();
+ value->scale = leaf_type->get_scale();
+ }
+}
+
+template <PrimitiveType Primitive>
+static Status fill_floating_point_variant_field(const Field& field, FieldWithDataType* value) {
+ const auto typed_value = field.get<Primitive>();
+ if (!std::isfinite(typed_value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point typed_value is not supported");
+ }
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ fill_variant_leaf_type_info(type, value);
+ return Status::OK();
+ case TYPE_FLOAT:
+ return fill_floating_point_variant_field<TYPE_FLOAT>(field, value);
+ case TYPE_DOUBLE:
+ return fill_floating_point_variant_field<TYPE_DOUBLE>(field, value);
+ case TYPE_TIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>())));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATE:
+ value->field = Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATE>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIME:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIME>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATEV2:
+ value->field =
+ Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIMEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_VARBINARY:
+ return Status::NotSupported("Parquet VARIANT binary typed_value is not supported");
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present);
+
+static Status serialize_field_to_json(const DataTypePtr& data_type, const Field& field,
+ std::string* json) {
+ MutableColumnPtr column = data_type->create_column();
+ column->insert(field);
+
+ auto json_column = ColumnString::create();
+ VectorBufferWriter writer(*json_column);
+ auto serde = data_type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ RETURN_IF_ERROR(serde->serialize_one_cell_to_json(*column, 0, writer, options));
+ writer.commit();
+ *json = json_column->get_data_at(0).to_string();
+ return Status::OK();
+}
+
+static Status scalar_typed_value_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(field_schema, field, &value, present));
+ if (!*present) {
+ return Status::OK();
+ }
+ if (value.field.is_null()) {
+ *json = "null";
+ return Status::OK();
+ }
+
+ DataTypePtr json_type;
+ if (value.base_scalar_type_id != PrimitiveType::INVALID_TYPE) {
+ json_type = DataTypeFactory::instance().create_data_type(value.base_scalar_type_id, false,
+ value.precision, value.scale);
+ } else {
+ json_type = remove_nullable(field_schema.data_type);
+ }
+ return serialize_field_to_json(json_type, value.field, json);
+}
+
+static Status resolve_variant_metadata(const FieldSchema& variant_field, const Struct& fields,
+ const std::string* inherited_metadata, std::string* metadata,
+ bool* has_metadata) {
+ *has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ *metadata = *inherited_metadata;
+ *has_metadata = true;
+ }
+
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], metadata, &metadata_present));
+ *has_metadata = metadata_present;
+ }
+ return Status::OK();
+}
+
+static Status variant_typed_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, std::string* typed_json,
+ bool* typed_present) {
+ *typed_present = false;
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ if (typed_value_idx < 0) {
+ return Status::OK();
+ }
+ return typed_value_to_json(variant_field.children[typed_value_idx], fields[typed_value_idx],
+ metadata, typed_json, typed_present);
+}
+
+static Status variant_residual_value_to_json(const FieldSchema& variant_field, const Struct& fields,
+ const std::string& metadata, bool has_metadata,
+ std::string* value_json, bool* value_present) {
+ *value_present = false;
+ const int value_idx = find_child_idx(variant_field, "value");
+ if (value_idx < 0) {
+ return Status::OK();
+ }
+
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, value_present));
+ if (!*value_present) {
+ return Status::OK();
+ }
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ return parquet::decode_variant_to_json(StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), value_json);
+}
+
+static Status merge_variant_value_and_typed_json(const std::string& value_json,
+ const std::string& typed_json, std::string* json) {
+ VariantMap value_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, PathInData(), &value_values));
+ VariantMap typed_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(typed_json, PathInData(), &typed_values));
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto root_value = value_values.find(PathInData());
+ if (root_value != value_values.end() && !is_empty_object_marker(root_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, PathInData()));
+ value_values.merge(std::move(typed_values));
+ return variant_map_to_json(std::move(value_values), json);
+}
+
+static Status variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ std::string metadata;
+ bool has_metadata = false;
+ RETURN_IF_ERROR(resolve_variant_metadata(variant_field, fields, inherited_metadata, &metadata,
+ &has_metadata));
+
+ std::string typed_json;
+ bool typed_present = false;
+ RETURN_IF_ERROR(variant_typed_value_to_json(variant_field, fields, metadata, &typed_json,
+ &typed_present));
+
+ std::string value_json;
+ bool value_present = false;
+ RETURN_IF_ERROR(variant_residual_value_to_json(variant_field, fields, metadata, has_metadata,
+ &value_json, &value_present));
+
+ if (value_present && typed_present) {
+ RETURN_IF_ERROR(merge_variant_value_and_typed_json(value_json, typed_json, json));
+ *present = true;
+ return Status::OK();
+ }
+
+ if (typed_present) {
+ *json = std::move(typed_json);
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+
+ *present = false;
+ return Status::OK();
+}
+
+static Status shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json, bool* present,
+ bool allow_scalar_typed_value_only_wrapper) {
+ if (is_variant_wrapper_field(field_schema, allow_scalar_typed_value_only_wrapper, true)) {
+ return variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ return typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+static Status typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ json->clear();
+ json->push_back('[');
+ for (size_t i = 0; i < elements.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(element_schema, elements[i], metadata, &element_json,
+ &element_present, true));
+ if (element_present) {
+ json->append(element_json);
+ } else {
+ return Status::Corruption("Parquet VARIANT array typed_value element is missing");
+ }
+ }
+ json->push_back(']');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ json->clear();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(typed_value_field.children[i], fields[i], metadata,
+ &child_json, &child_present, false));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ first = false;
+ }
+ json->push_back('}');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ switch (typed_type->get_primitive_type()) {
+ case TYPE_STRUCT:
+ return typed_struct_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_ARRAY:
+ return typed_array_to_json(typed_value_field, field, metadata, json, present);
+ default:
+ return scalar_typed_value_to_json(typed_value_field, field, json, present);
+ }
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present);
+
+static Status variant_to_variant_map(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ VariantMap value_values;
+ bool value_present = false;
+ const PathInData current_path = path->build();
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ std::string value_json;
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, current_path, &value_values));
+ }
+ }
+
+ VariantMap typed_values;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_variant_map(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, path,
+ &typed_values, &typed_present));
+ }
+
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto current_value = value_values.find(current_path);
+ if (value_present && typed_present && current_value != value_values.end() &&
+ !is_empty_object_marker(current_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, current_path));
+ values->merge(std::move(value_values));
+ values->merge(std::move(typed_values));
+ *present = value_present || typed_present;
+ return Status::OK();
+}
+
+static Status shredded_field_to_variant_map(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (is_variant_wrapper_field(field_schema, false, true)) {
+ return variant_to_variant_map(field_schema, field, &metadata, path, values, present);
+ }
+ return typed_value_to_variant_map(field_schema, field, metadata, path, values, present);
+}
+
+static Status append_typed_field_to_variant_map(const FieldSchema& typed_value_field,
+ const Field& field, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static Status typed_array_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (is_direct_variant_leaf_type(typed_value_field.data_type)) {
+ return append_typed_field_to_variant_map(typed_value_field, field, path, values, present);
+ }
+
+ std::string value_json;
+ RETURN_IF_ERROR(typed_value_to_json(typed_value_field, field, metadata, &value_json, present));
+ if (*present) {
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, path->build(), values));
+ }
+ return Status::OK();
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& fields = field.get<TYPE_STRUCT>();
+ *present = true;
+ bool has_present_child = false;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ path->append(typed_value_field.children[i].name, false);
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_variant_map(typed_value_field.children[i], fields[i],
+ metadata, path, values, &child_present));
+ has_present_child |= child_present;
+ path->pop_back();
+ }
+ if (!has_present_child) {
+ RETURN_IF_ERROR(insert_empty_object_marker(path->build(), values));
+ }
+ return Status::OK();
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ return typed_array_to_variant_map(typed_value_field, field, metadata, path, values,
+ present);
+ }
+
+ return append_typed_field_to_variant_map(typed_value_field, field, path, values, present);
+}
+
+static Status append_direct_typed_column_to_batch(const FieldSchema& field_schema,
+ const IColumn& column, size_t start, size_t rows,
+ PathInDataBuilder* path, ColumnVariant* batch,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids,
+ std::vector<const NullMap*> parent_null_maps) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return Status::OK();
+ }
+
+ const IColumn* value_column = &column;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ parent_null_maps.push_back(&nullable_column->get_null_map_data());
+ value_column = &nullable_column->get_nested_column();
+ }
+
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ const auto& typed_struct = assert_cast<const ColumnStruct&>(*value_column);
+ return append_direct_typed_column_to_batch(
+ field_schema.children[typed_value_idx], typed_struct.get_column(typed_value_idx),
+ start, rows, path, batch, false, column_ids, parent_null_maps);
+ }
+
Review Comment:
The direct typed path does not preserve present empty typed objects. The row-wise path below marks a present struct with no present children via `insert_empty_object_marker()`, but this direct path only copies selected leaf subcolumns. For a typed-only layout such as `typed_value.obj { optional int32 x; }`, a row where `obj` is present and `x` is absent/null can take the direct path and only add nullable `obj.x`; the empty object marker is never emitted, so the object can disappear or become `obj.x = null` instead of `obj = {}`. Please either make the direct path ineligible for nullable typed structs that need empty-object markers, or emit equivalent markers, and add coverage for a present empty typed struct.
##########
be/src/format/table/iceberg/types.cpp:
##########
@@ -170,6 +170,8 @@ std::unique_ptr<PrimitiveType> Types::from_primitive_string(const std::string& t
return std::make_unique<UUIDType>();
} else if (lower_type_string == "binary") {
return std::make_unique<BinaryType>();
+ } else if (lower_type_string == "variant") {
Review Comment:
This exposes Iceberg `variant` to the BE Iceberg schema model, but the Iceberg Parquet write path still cannot convert it. `VParquetTransformer::_parse_schema()` calls `ArrowSchemaUtil::convert()`, and `be/src/format/table/iceberg/arrow_schema_util.cpp` has no `TypeID::VARIANT` case, so an Iceberg table with a VARIANT column can be mapped for reads but `INSERT INTO`/write planning will fail at runtime with `Unsupported field type:variant`. If writes are intentionally unsupported, FE should reject Iceberg VARIANT writes before sending them to BE; otherwise the Arrow/Iceberg schema conversion needs matching VARIANT support.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467036164
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467834442
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468154303
<details>
<summary>TPC-H: <b>Total hot run time: 31026 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 01137d15607e99a312eb4d0648a28b499cd2033f, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17603 3876 3824 3824
q2 q3 10762 1450 820 820
q4 4694 478 353 353
q5 7638 2327 2112 2112
q6 388 187 144 144
q7 971 777 660 660
q8 9439 1783 1552 1552
q9 6961 4946 4921 4921
q10 6433 2132 1771 1771
q11 436 272 253 253
q12 694 429 297 297
q13 18230 3463 2817 2817
q14 267 259 237 237
q15 q16 816 771 698 698
q17 954 932 1069 932
q18 6821 5676 5513 5513
q19 1363 1233 1101 1101
q20 513 421 266 266
q21 5892 2626 2448 2448
q22 449 363 307 307
Total cold run time: 101324 ms
Total hot run time: 31026 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4370 4173 4186 4173
q2 q3 4545 4926 4311 4311
q4 2144 2203 1404 1404
q5 4507 4361 4785 4361
q6 270 213 155 155
q7 2043 1831 1582 1582
q8 2549 2331 2191 2191
q9 7788 7821 7819 7819
q10 4556 4482 4058 4058
q11 789 441 420 420
q12 713 734 525 525
q13 3460 3630 3112 3112
q14 298 328 292 292
q15 q16 707 728 638 638
q17 1385 1382 1374 1374
q18 8098 7295 7062 7062
q19 1146 1094 1078 1078
q20 2215 2232 1932 1932
q21 5394 4710 4644 4644
q22 524 477 413 413
Total cold run time: 57501 ms
Total hot run time: 51544 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470181536
<details>
<summary>TPC-DS: <b>Total hot run time: 168382 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit b7402399a565dc0a3e621ecb4d61cfd48a4eb2fc, data reload: false
query5 4321 657 505 505
query6 332 219 203 203
query7 4214 575 294 294
query8 328 235 232 232
query9 8818 4000 4017 4000
query10 441 351 300 300
query11 5776 2426 2195 2195
query12 183 130 126 126
query13 1271 603 435 435
query14 5983 5378 5036 5036
query14_1 4360 4374 4317 4317
query15 207 203 181 181
query16 1009 442 424 424
query17 1144 710 599 599
query18 2731 475 352 352
query19 213 204 163 163
query20 138 134 128 128
query21 218 135 116 116
query22 13609 13533 13287 13287
query23 17065 16409 16068 16068
query23_1 16203 16105 16209 16105
query24 7543 1787 1265 1265
query24_1 1314 1292 1320 1292
query25 562 486 421 421
query26 1294 308 169 169
query27 2735 537 345 345
query28 4422 1956 1950 1950
query29 1032 630 499 499
query30 335 244 201 201
query31 1100 1046 928 928
query32 86 75 71 71
query33 534 370 297 297
query34 1183 1162 654 654
query35 755 769 667 667
query36 1296 1293 1160 1160
query37 153 108 104 104
query38 3214 3154 3074 3074
query39 918 920 897 897
query39_1 878 892 872 872
query40 249 145 128 128
query41 68 64 64 64
query42 112 114 110 110
query43 322 323 284 284
query44
query45 279 202 200 200
query46 1067 1237 745 745
query47 2298 2335 2248 2248
query48 381 409 290 290
query49 637 491 400 400
query50 983 338 258 258
query51 4332 4280 4166 4166
query52 110 106 101 101
query53 259 282 207 207
query54 315 283 273 273
query55 96 92 88 88
query56 309 320 323 320
query57 1409 1401 1266 1266
query58 303 269 273 269
query59 1526 1671 1396 1396
query60 319 321 312 312
query61 186 178 186 178
query62 681 623 553 553
query63 251 207 216 207
query64 2457 864 713 713
query65
query66 1663 499 393 393
query67 30057 29894 29684 29684
query68
query69 459 342 301 301
query70 1046 995 941 941
query71 315 291 278 278
query72 2967 2703 1984 1984
query73 840 728 423 423
query74 5051 4905 4705 4705
query75 2675 2587 2258 2258
query76 2305 1180 801 801
query77 401 430 339 339
query78 12058 11971 11548 11548
query79 1242 1033 754 754
query80 584 547 468 468
query81 459 281 236 236
query82 240 163 122 122
query83 357 272 244 244
query84 257 141 109 109
query85 863 530 461 461
query86 396 373 315 315
query87 3384 3456 3256 3256
query88 3547 2689 2652 2652
query89 426 388 337 337
query90 2179 188 189 188
query91 186 168 142 142
query92 78 80 74 74
query93 1449 1528 873 873
query94 553 353 314 314
query95 696 392 348 348
query96 1040 783 329 329
query97 2704 2688 2569 2569
query98 256 243 244 243
query99 1112 1098 975 975
Total cold run time: 251753 ms
Total hot run time: 168382 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3231360614
##########
be/src/format/parquet/vparquet_column_reader.h:
##########
@@ -460,6 +463,61 @@ class StructColumnReader : public ParquetColumnReader {
//Need to use vector instead of set,see `get_rep_level()` for the reason.
};
+class VariantColumnReader : public ParquetColumnReader {
+ ENABLE_FACTORY_CREATOR(VariantColumnReader)
+public:
+ VariantColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz,
+ io::IOContext* io_ctx)
+ : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {}
+ ~VariantColumnReader() override { close(); }
+
+ Status init(io::FileReaderSPtr file, FieldSchema* field, const tparquet::RowGroup& row_group,
+ size_t max_buf_size, std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection, const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids);
+ Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
+ FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
+ bool is_dict_filter, int64_t real_column_size = -1) override;
+
+ const std::vector<level_t>& get_rep_level() const override {
+ return _struct_reader->get_rep_level();
+ }
+ const std::vector<level_t>& get_def_level() const override {
+ return _struct_reader->get_def_level();
+ }
+ ColumnStatistics column_statistics() override { return _struct_reader->column_statistics(); }
+ void close() override {}
+
+ void reset_filter_map_index() override { _struct_reader->reset_filter_map_index(); }
+
+private:
+ Status _variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) const;
+ Status _typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json,
+ bool* present) const;
+ Status _shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json,
+ bool* present) const;
+ Status _typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json) const;
+ Status _typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json) const;
+ Status _field_to_json(const FieldSchema& field_schema, const Field& field, std::string* json,
+ bool* present) const;
+ Status _array_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const DataTypeArray* array_type, std::string* json) const;
+ Status _struct_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json) const;
+ Status _get_binary_field(const Field& field, std::string* value, bool* present) const;
+
+ std::unique_ptr<FieldSchema> _variant_struct_field;
+ DataTypePtr _variant_struct_type;
Review Comment:
为什么要引入一个struct type, 不引入有什么问题
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437462681
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434015028
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437987677
<details>
<summary>TPC-H: <b>Total hot run time: 29487 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit bcda6244eb29152d7533075da24e9c6b6dc1990f, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17824 3912 3867 3867
q2 q3 10706 869 599 599
q4 4662 464 347 347
q5 7460 1342 1136 1136
q6 185 171 138 138
q7 902 935 765 765
q8 9322 1367 1231 1231
q9 5582 5383 5346 5346
q10 6258 2087 1802 1802
q11 466 272 269 269
q12 635 420 287 287
q13 18087 3340 2788 2788
q14 294 281 260 260
q15 q16 864 879 784 784
q17 1001 1084 699 699
q18 6502 5704 5547 5547
q19 1146 1302 1057 1057
q20 484 404 267 267
q21 4564 2412 1966 1966
q22 477 408 332 332
Total cold run time: 97421 ms
Total hot run time: 29487 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4750 4781 4837 4781
q2 q3 4662 4756 4245 4245
q4 2224 2165 1407 1407
q5 4968 5020 5291 5020
q6 202 170 138 138
q7 2016 1821 1607 1607
q8 3401 3070 3099 3070
q9 8412 8384 8400 8384
q10 4490 4459 4242 4242
q11 606 421 406 406
q12 693 752 529 529
q13 3201 3668 2877 2877
q14 296 300 269 269
q15 q16 754 831 827 827
q17 1347 1310 1256 1256
q18 7885 7167 7124 7124
q19 1145 1150 1153 1150
q20 2239 2229 1949 1949
q21 6103 5353 4890 4890
q22 538 498 417 417
Total cold run time: 59932 ms
Total hot run time: 54588 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4439912155
<details>
<summary>TPC-H: <b>Total hot run time: 29463 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 1beb1746c3e4fbef9cb87898dc837d92107b692a, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17696 3885 3825 3825
q2 q3 10701 857 589 589
q4 4661 454 343 343
q5 7448 1320 1140 1140
q6 191 164 140 140
q7 907 924 755 755
q8 9422 1335 1265 1265
q9 6168 5320 5335 5320
q10 6322 2074 1802 1802
q11 471 265 259 259
q12 683 404 286 286
q13 18181 3245 2761 2761
q14 295 283 268 268
q15 q16 898 874 790 790
q17 1008 1036 743 743
q18 6475 5735 5646 5646
q19 1339 1264 1044 1044
q20 530 388 256 256
q21 4774 2327 1904 1904
q22 487 389 327 327
Total cold run time: 98657 ms
Total hot run time: 29463 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4672 4475 4547 4475
q2 q3 4653 4778 4189 4189
q4 2123 2154 1407 1407
q5 4967 4985 5280 4985
q6 191 166 131 131
q7 2057 1827 1623 1623
q8 3350 3123 3120 3120
q9 8435 8593 8382 8382
q10 4478 4488 4260 4260
q11 604 410 400 400
q12 693 748 513 513
q13 3220 3572 2984 2984
q14 306 306 282 282
q15 q16 747 814 729 729
q17 1348 1311 1270 1270
q18 8000 7125 7066 7066
q19 1122 1163 1160 1160
q20 2221 2228 1954 1954
q21 6143 5304 4749 4749
q22 528 488 399 399
Total cold run time: 59858 ms
Total hot run time: 54078 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4439646170
# BE UT Coverage Report
Increment line coverage `34.98% (629/1798)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/d04698170e31bbb586bf03c0fb6c3dcb85f8d890_d04698170e31bbb586bf03c0fb6c3dcb85f8d890/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/d04698170e31bbb586bf03c0fb6c3dcb85f8d890_d04698170e31bbb586bf03c0fb6c3dcb85f8d890/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.54% (20658/38586) |
| Line Coverage | 37.19% (195520/525713) |
| Region Coverage | 33.64% (153109/455112) |
| Branch Coverage | 34.61% (66845/193121) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441806929
<details>
<summary>TPC-DS: <b>Total hot run time: 170586 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 7723e445c78e8ca2c60829c8910f6288a1ec066d, data reload: false
query5 4330 648 533 533
query6 324 226 199 199
query7 4212 591 303 303
query8 325 238 226 226
query9 8813 4021 4021 4021
query10 454 328 296 296
query11 5816 2331 2221 2221
query12 193 128 124 124
query13 1298 628 435 435
query14 5890 5400 5002 5002
query14_1 4331 4322 4330 4322
query15 210 203 176 176
query16 981 459 440 440
query17 940 754 608 608
query18 2454 478 347 347
query19 212 206 170 170
query20 138 128 130 128
query21 213 135 117 117
query22 13618 13484 13434 13434
query23 17079 16367 16066 16066
query23_1 16235 16123 16031 16031
query24 7359 1749 1392 1392
query24_1 1337 1341 1345 1341
query25 574 507 462 462
query26 1326 335 173 173
query27 2731 600 355 355
query28 4440 1980 1984 1980
query29 1022 656 561 561
query30 299 237 197 197
query31 1118 1064 935 935
query32 90 77 76 76
query33 517 358 289 289
query34 1188 1145 644 644
query35 751 781 665 665
query36 1319 1376 1146 1146
query37 159 100 95 95
query38 3227 3128 3051 3051
query39 927 932 894 894
query39_1 879 900 880 880
query40 227 161 135 135
query41 69 65 61 61
query42 113 110 107 107
query43 319 323 286 286
query44
query45 211 200 193 193
query46 1064 1223 735 735
query47 2322 2381 2179 2179
query48 405 400 303 303
query49 650 527 440 440
query50 703 279 213 213
query51 4293 4239 4252 4239
query52 104 108 94 94
query53 246 283 204 204
query54 314 272 246 246
query55 95 88 92 88
query56 290 317 297 297
query57 1432 1418 1305 1305
query58 294 268 266 266
query59 1531 1590 1423 1423
query60 350 330 330 330
query61 166 163 149 149
query62 676 645 561 561
query63 241 198 213 198
query64 2494 837 693 693
query65
query66 1766 510 396 396
query67 29933 29816 29774 29774
query68
query69 454 346 305 305
query70 1032 1023 982 982
query71 313 279 272 272
query72 2973 2686 2484 2484
query73 862 750 420 420
query74 5041 4902 4717 4717
query75 2785 2676 2371 2371
query76 2303 1144 762 762
query77 418 431 360 360
query78 12881 12888 12279 12279
query79 1488 991 767 767
query80 709 616 522 522
query81 457 287 242 242
query82 1359 162 130 130
query83 380 286 261 261
query84 316 147 122 122
query85 971 617 536 536
query86 397 334 345 334
query87 3441 3357 3208 3208
query88 3546 2689 2664 2664
query89 481 393 337 337
query90 1951 180 176 176
query91 182 174 140 140
query92 80 79 76 76
query93 946 958 555 555
query94 521 338 294 294
query95 685 378 360 360
query96 1035 788 310 310
query97 2674 2683 2548 2548
query98 242 230 231 230
query99 1108 1107 970 970
Total cold run time: 252527 ms
Total hot run time: 170586 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434317705
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4436900541
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441123596
<details>
<summary>TPC-H: <b>Total hot run time: 29773 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 9084f82a0ae8c8ca055f19549ac56d6fb71a510c, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17623 3959 3937 3937
q2 q3 10711 875 614 614
q4 4667 466 349 349
q5 7511 1350 1191 1191
q6 195 175 143 143
q7 943 960 765 765
q8 9713 1439 1312 1312
q9 6264 5363 5331 5331
q10 6313 2075 1810 1810
q11 490 283 272 272
q12 692 427 297 297
q13 18198 3630 2730 2730
q14 296 288 264 264
q15 q16 921 851 794 794
q17 967 1037 749 749
q18 6444 5722 5587 5587
q19 1218 1259 1067 1067
q20 523 407 274 274
q21 4599 2372 1954 1954
q22 479 407 333 333
Total cold run time: 98767 ms
Total hot run time: 29773 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4627 4636 4597 4597
q2 q3 4667 4779 4196 4196
q4 2172 2244 1433 1433
q5 5008 5072 5315 5072
q6 192 165 130 130
q7 2073 1815 1598 1598
q8 3374 3118 3150 3118
q9 8492 8527 8448 8448
q10 4473 4464 4259 4259
q11 599 454 400 400
q12 714 751 524 524
q13 3269 3642 2888 2888
q14 326 312 278 278
q15 q16 789 770 682 682
q17 1391 1369 1270 1270
q18 7838 7089 7251 7089
q19 1188 1184 1163 1163
q20 2251 2224 1957 1957
q21 6110 5344 4849 4849
q22 537 476 407 407
Total cold run time: 60090 ms
Total hot run time: 54358 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437711668
<details>
<summary>TPC-DS: <b>Total hot run time: 170408 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit cb90a9e5d9e7319be77f506d54888d62c5dc36b7, data reload: false
query5 4330 662 525 525
query6 327 223 204 204
query7 4226 537 317 317
query8 328 236 232 232
query9 8821 4051 4000 4000
query10 460 336 301 301
query11 6055 2358 2194 2194
query12 182 125 124 124
query13 1279 618 445 445
query14 6576 5341 5050 5050
query14_1 4330 4335 4342 4335
query15 212 205 188 188
query16 1006 439 414 414
query17 1125 750 619 619
query18 2495 490 353 353
query19 215 203 183 183
query20 141 136 139 136
query21 214 143 117 117
query22 13559 13496 13333 13333
query23 17069 16441 16593 16441
query23_1 16359 16392 16395 16392
query24 7553 1817 1364 1364
query24_1 1354 1355 1342 1342
query25 586 512 465 465
query26 1330 304 177 177
query27 2693 615 346 346
query28 4477 1974 1969 1969
query29 1033 671 539 539
query30 299 237 199 199
query31 1111 1083 940 940
query32 96 80 74 74
query33 590 347 295 295
query34 1137 1108 630 630
query35 793 792 664 664
query36 1327 1325 1114 1114
query37 148 98 85 85
query38 3193 3121 3103 3103
query39 928 915 925 915
query39_1 872 863 853 853
query40 234 151 134 134
query41 62 59 57 57
query42 109 108 104 104
query43 321 329 284 284
query44
query45 206 202 190 190
query46 1041 1172 701 701
query47 2298 2355 2158 2158
query48 396 399 273 273
query49 636 522 419 419
query50 685 279 215 215
query51 4310 4328 4176 4176
query52 106 103 94 94
query53 258 268 201 201
query54 309 270 248 248
query55 96 89 83 83
query56 308 307 302 302
query57 1430 1358 1326 1326
query58 324 271 274 271
query59 1521 1670 1389 1389
query60 365 332 328 328
query61 205 162 173 162
query62 667 630 562 562
query63 235 201 205 201
query64 2425 838 666 666
query65
query66 1715 528 390 390
query67 30116 30059 29851 29851
query68
query69 466 334 306 306
query70 974 1000 958 958
query71 310 278 268 268
query72 2953 2814 2559 2559
query73 818 716 431 431
query74 5101 4889 4716 4716
query75 2777 2646 2328 2328
query76 2292 1125 765 765
query77 402 429 356 356
query78 12866 12982 12245 12245
query79 1473 951 756 756
query80 758 591 484 484
query81 467 282 235 235
query82 1336 160 124 124
query83 340 268 254 254
query84 261 146 110 110
query85 911 584 529 529
query86 422 318 328 318
query87 3414 3329 3230 3230
query88 3534 2682 2632 2632
query89 432 376 336 336
query90 1802 187 178 178
query91 177 180 144 144
query92 78 80 73 73
query93 958 950 549 549
query94 565 342 317 317
query95 657 474 348 348
query96 1047 759 320 320
query97 2701 2678 2577 2577
query98 246 237 227 227
query99 1126 1107 971 971
Total cold run time: 253876 ms
Total hot run time: 170408 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3235015294
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,549 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <unordered_set>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+void append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ json->append("null");
+ return;
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, dictionary_size + 1, offset_size,
+ "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ std::unordered_set<std::string> dictionary_entries;
+ dictionary_entries.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ std::string entry(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
+ if (!dictionary_entries.insert(entry).second) {
+ return Status::Corruption(
+ "Invalid Parquet VARIANT metadata duplicate dictionary key {}", entry);
+ }
+ result->dictionary.emplace_back(std::move(entry));
+ }
+ return Status::OK();
+}
+
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next) {
+ switch (primitive_header) {
+ case 0:
+ json->append("null");
+ *next = ptr;
+ return Status::OK();
+ case 1:
+ json->append("true");
+ *next = ptr;
+ return Status::OK();
+ case 2:
+ json->append("false");
+ *next = ptr;
+ return Status::OK();
+ case 3:
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "int8 value"));
+ json->append(std::to_string(static_cast<int8_t>(*ptr)));
+ *next = ptr + 1;
+ return Status::OK();
+ case 4:
+ RETURN_IF_ERROR(require_available(ptr, end, 2, "int16 value"));
+ json->append(std::to_string(read_signed_le(ptr, 2)));
+ *next = ptr + 2;
+ return Status::OK();
+ case 5:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "int32 value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 6:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "int64 value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 7: {
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "double value"));
+ uint64_t bits = read_unsigned_le(ptr, 8);
+ double value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 8;
+ return Status::OK();
+ }
+ case 8:
+ case 9:
+ case 10: {
+ int value_size = primitive_header == 8 ? 4 : primitive_header == 9 ? 8 : 16;
+ RETURN_IF_ERROR(require_available(ptr, end, 1 + value_size, "decimal value"));
+ int scale = static_cast<int8_t>(*ptr++);
+ __int128 unscaled = 0;
+ if (value_size == 16) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ if ((ptr[15] & 0x80) != 0) {
+ unscaled = -static_cast<__int128>((~unsigned_value) + 1);
Review Comment:
Fixed by decoding signed decimal128 payloads with bounded two-complement arithmetic instead of negating the 2^127 magnitude. Added DecodeDecimal128MinimumValue coverage and verified locally with `./run-be-ut.sh --run --filter=ParquetVariantReaderTest.*`, `build-support/check-format.sh`, and `git diff --check`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3229421164
##########
be/src/format/table/hive/hive_parquet_nested_column_utils.cpp:
##########
@@ -18,20 +18,172 @@
#include "format/table/hive/hive_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_name(const FieldSchema& field_schema, std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+bool extract_nested_column_ids_by_name(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+bool extract_variant_nested_column_ids(const FieldSchema& variant_field,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_name(variant_field, "typed_value");
+ bool has_child_columns = false;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ has_child_columns = true;
+ continue;
+ }
+
+ add_variant_metadata(variant_field, column_ids);
+ bool found_typed_path = false;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ found_typed_path = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ found_typed_path = extract_nested_column_ids_by_name(*typed_child, child_paths,
+ column_ids);
+ }
+ if (found_typed_path) {
+ column_ids.insert(typed_value->get_column_id());
+ }
+ }
+ }
+
+ if (!found_typed_path) {
+ add_variant_value(variant_field, column_ids);
Review Comment:
Handled by documenting the Parquet VARIANT shredding invariant and tightening the regression observable. Per the Parquet Variant Shredding spec, object field residual fallback is stored at typed_value.<field>.value, while the top-level value column stores non-shredded object fields and must not contain shredded field keys. The regression now asserts that v['metric'] reads v.typed_value.metric.typed_value and v.typed_value.metric.value, does not read v.value, and does not read sibling v.typed_value.name. Source: https://parquet.apache.org/docs/file-format/types/variantshredding/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4436318141
# FE Regression Coverage Report
Increment line coverage ` 0.00% (0/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_7c992131eb1a6f8bbd285fd83cecec07b53e5bfe_merge_fe/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_7c992131eb1a6f8bbd285fd83cecec07b53e5bfe_merge_fe/report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3233481209
##########
be/src/format/parquet/vparquet_reader.cpp:
##########
@@ -432,11 +435,111 @@ Status ParquetReader::on_before_init_reader(ReaderInitContext* ctx) {
RETURN_IF_ERROR(get_file_metadata_schema(&field_desc));
RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
ctx->tuple_descriptor, *field_desc, ctx->table_info_node));
+ auto column_id_result = _create_column_ids_by_name(field_desc, ctx->tuple_descriptor);
+ ctx->column_ids = std::move(column_id_result.column_ids);
+ ctx->filter_column_ids = std::move(column_id_result.filter_column_ids);
}
return Status::OK();
}
+ColumnIdResult ParquetReader::_create_column_ids_by_name(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<std::string, const FieldSchema*> table_col_name_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) {
+ continue;
+ }
+ table_col_name_to_field_schema_map[field_schema->lower_case_name] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ process_nested_access_paths(
+ parquet_field, access_paths, out_ids,
+ [](const FieldSchema* field) { return field->get_column_id(); },
+ [](const FieldSchema* field) { return field->get_max_column_id(); },
+ HiveParquetNestedColumnUtils::extract_nested_column_ids);
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_name_to_field_schema_map.find(slot->col_name_lower_case());
+ if (it == table_col_name_to_field_schema_map.end()) {
+ continue;
+ }
+ auto field_schema = it->second;
+
Review Comment:
Fixed in 918c87c5454a. The shared nested access-path helper now falls back to the full field column-id range when a complex/VARIANT slot has no usable DATA/META access paths, so selecting id plus full v keeps the VARIANT leaves instead of pruning them all. This helper is used by local Parquet, Hive Parquet including the top-level-column-index path, and Iceberg Parquet. Added BE coverage for empty/ignored access paths and a regression query selecting id with full root v. Local verification: ./build.sh --be; ./run-be-ut.sh --run --filter=NestedColumnAccessHelperTest.*:ParquetVariantReaderTest.*; ./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant; build-support/check-format.sh; git diff --check.
##########
be/src/format/table/hive_reader.cpp:
##########
@@ -328,7 +328,7 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
// primitive (non-nested) types
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
Review Comment:
Fixed in 918c87c5454a. The shared nested access-path helper now falls back to the full field column-id range when a complex/VARIANT slot has no usable DATA/META access paths, so selecting id plus full v keeps the VARIANT leaves instead of pruning them all. This helper is used by local Parquet, Hive Parquet including the top-level-column-index path, and Iceberg Parquet. Added BE coverage for empty/ignored access paths and a regression query selecting id with full root v. Local verification: ./build.sh --be; ./run-be-ut.sh --run --filter=NestedColumnAccessHelperTest.*:ParquetVariantReaderTest.*; ./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant; build-support/check-format.sh; git diff --check.
##########
be/src/format/table/iceberg_reader.cpp:
##########
@@ -344,7 +344,7 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
auto field_schema = it->second;
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
- slot->col_type() != TYPE_MAP)) {
+ slot->col_type() != TYPE_MAP && slot->col_type() != TYPE_VARIANT)) {
Review Comment:
Fixed in 918c87c5454a. The shared nested access-path helper now falls back to the full field column-id range when a complex/VARIANT slot has no usable DATA/META access paths, so selecting id plus full v keeps the VARIANT leaves instead of pruning them all. This helper is used by local Parquet, Hive Parquet including the top-level-column-index path, and Iceberg Parquet. Added BE coverage for empty/ignored access paths and a regression query selecting id with full root v. Local verification: ./build.sh --be; ./run-be-ut.sh --run --filter=NestedColumnAccessHelperTest.*:ParquetVariantReaderTest.*; ./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant; build-support/check-format.sh; git diff --check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4439007042
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440175774
run compile
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441351545
<details>
<summary>TPC-DS: <b>Total hot run time: 168960 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 7aef30cdbcdebd9735846948a34f710fcb054f2b, data reload: false
query5 4324 653 516 516
query6 336 231 213 213
query7 4230 572 323 323
query8 329 244 246 244
query9 8830 4055 4014 4014
query10 473 356 302 302
query11 6046 2421 2196 2196
query12 188 132 131 131
query13 1319 637 435 435
query14 6883 5380 5064 5064
query14_1 4398 4369 4394 4369
query15 216 206 191 191
query16 1001 478 438 438
query17 1153 774 642 642
query18 2737 487 359 359
query19 223 249 178 178
query20 142 139 126 126
query21 213 143 121 121
query22 13553 13569 13346 13346
query23 17177 16188 16493 16188
query23_1 16248 16293 16199 16199
query24 7652 1800 1373 1373
query24_1 1425 1346 1335 1335
query25 562 496 421 421
query26 1294 306 170 170
query27 2743 583 340 340
query28 4445 1992 1940 1940
query29 961 647 497 497
query30 298 238 198 198
query31 1143 1077 930 930
query32 79 75 69 69
query33 521 374 288 288
query34 1154 1166 637 637
query35 761 760 662 662
query36 1342 1345 1081 1081
query37 148 100 85 85
query38 3130 3144 3055 3055
query39 920 915 874 874
query39_1 877 865 878 865
query40 233 156 134 134
query41 61 59 63 59
query42 108 103 102 102
query43 320 319 282 282
query44
query45 216 195 190 190
query46 1062 1207 715 715
query47 2246 2238 2075 2075
query48 398 422 294 294
query49 627 524 417 417
query50 705 279 216 216
query51 4313 4278 4135 4135
query52 103 105 91 91
query53 253 275 200 200
query54 309 261 261 261
query55 93 88 85 85
query56 291 311 302 302
query57 1407 1377 1273 1273
query58 291 262 249 249
query59 1498 1628 1398 1398
query60 344 330 366 330
query61 156 151 156 151
query62 674 610 535 535
query63 239 197 200 197
query64 2343 822 682 682
query65
query66 1683 505 393 393
query67 30020 29923 29225 29225
query68
query69 450 340 306 306
query70 1017 1009 993 993
query71 313 265 271 265
query72 2914 2760 2442 2442
query73 856 780 413 413
query74 5111 4888 4727 4727
query75 2765 2676 2319 2319
query76 2301 1115 749 749
query77 417 424 353 353
query78 12923 12892 12345 12345
query79 1372 1016 723 723
query80 699 601 501 501
query81 449 284 237 237
query82 1347 162 126 126
query83 365 284 277 277
query84 260 151 116 116
query85 938 600 529 529
query86 377 349 321 321
query87 3448 3338 3183 3183
query88 3541 2674 2708 2674
query89 428 387 328 328
query90 1817 179 179 179
query91 180 164 135 135
query92 76 76 67 67
query93 951 984 568 568
query94 550 332 292 292
query95 671 369 428 369
query96 1098 753 317 317
query97 2715 2670 2551 2551
query98 235 226 221 221
query99 1121 1088 970 970
Total cold run time: 254139 ms
Total hot run time: 168960 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "raghav-reglobe (via GitHub)" <gi...@apache.org>.
raghav-reglobe commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-5055274704
We've been running this branch rebased onto current master (ab043a43) against Iceberg V3 tables with shredded VARIANT parquet, and hit an interaction with the new file scanner: with `enable_file_scanner_v2=true` (the master default), any scan touching a VARIANT-bearing file fails at open with
```
[CORRUPTION]Failed to open parquet file: Metadata contains Thrift LogicalType that is not recognized
```
format_v2's footer path goes through arrow parquet-cpp (17.0.0 in thirdparty), which predates the parquet VARIANT LogicalType — and this PR's read support lives in the v1 reader tree, so v2 has no path for these files even past the footer. With `enable_file_scanner_v2=false` everything in this branch works well for us (bracket access, predicates on shredded columns, extraction — we've verified against fairly large shredded datasets).
Is scanner-v2 integration planned as part of this PR? Happy to help — either the small interim guard (routing VARIANT-bearing scans to v1 in `_should_use_file_scanner_v2`, same shape as the existing WAL/JNI/Lance exclusions, so the default-on v2 doesn't crash) or contributing toward proper v2 reader support if you'd take it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4435215792
# BE UT Coverage Report
Increment line coverage `6.57% (86/1308)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/bf0c548be5e55a9d9d6cd8e8b9643ce3eee1af21_bf0c548be5e55a9d9d6cd8e8b9643ce3eee1af21/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/bf0c548be5e55a9d9d6cd8e8b9643ce3eee1af21_bf0c548be5e55a9d9d6cd8e8b9643ce3eee1af21/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.48% (20638/38591) |
| Line Coverage | 37.12% (195030/525439) |
| Region Coverage | 33.53% (152460/454732) |
| Branch Coverage | 34.48% (66491/192862) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3230284680
##########
be/src/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp:
##########
@@ -18,21 +18,241 @@
#include "format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <iostream>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
Review Comment:
Handled in the latest head. The Iceberg helper now mirrors the Hive/local typed-value-only pruning behavior. The regression includes a typed-only nested profile assertion that reads v.typed_value.nested.typed_value.x and does not read v.typed_value.nested.value or top-level v.value.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1125,382 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
+ case TYPE_FLOAT: {
+ const auto value = field.get<TYPE_FLOAT>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_DOUBLE: {
+ const auto value = field.get<TYPE_DOUBLE>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_TIMEV2:
+ json->append(std::to_string(field.get<TYPE_TIMEV2>()));
+ return Status::OK();
+ case TYPE_DATE:
+ case TYPE_DATETIME:
Review Comment:
Handled in the latest head. Shredded temporal typed values now use Parquet VARIANT raw numeric semantics: date as epoch day, time as microseconds, and timestamp/timestamptz as UTC epoch microseconds. The regression joins unshredded and shredded temporal files and asserts date/time/timestamp parity, with profile checks proving only the typed temporal leaves are read.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3230283458
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1091,368 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
Review Comment:
Handled in the latest head. The shredded typed-value path now normalizes TYPE_FLOAT/TYPE_DOUBLE with std::isfinite() and emits JSON null for non-finite values, matching the unshredded decoder. The typed-only regression asserts cast(v[f] as string) is null for the non-finite case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440818129
# FE UT Coverage Report
Increment line coverage ` 25.00% (1/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_918c87c5454a4a3edef6fd9d50ea1e15b8d35c98/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_918c87c5454a4a3edef6fd9d50ea1e15b8d35c98/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3234146086
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +119,753 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
Review Comment:
Fixed in current head `9084f82a0ae8c8ca055f19549ac56d6fb71a510c`. Wrapper detection is now context-aware for nested typed-only structs: direct typed-only object traversal does not unwrap primitive user fields named `typed_value` or `value`, while array-element scalar typed wrappers remain supported. Added BE coverage in `ParquetVariantReaderTest.DirectTypedOnlyKeepsNestedStructuralNameUserKeys` plus Hive/Iceberg pruning tests for nested structural-name user keys. Verified locally with `./run-be-ut.sh --run --filter="ParquetVariantReaderTest.*:NestedColumnAccessHelperTest.*"`, `./build.sh --be`, `./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant`, `build-support/check-format.sh`, and `git diff --check`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440930695
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441221907
<details>
<summary>TPC-DS: <b>Total hot run time: 170549 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 9084f82a0ae8c8ca055f19549ac56d6fb71a510c, data reload: false
query5 4300 653 515 515
query6 322 231 199 199
query7 4329 561 307 307
query8 355 241 228 228
query9 8847 4077 4060 4060
query10 459 348 304 304
query11 5779 2416 2227 2227
query12 181 127 124 124
query13 1329 624 455 455
query14 6461 5336 5069 5069
query14_1 4390 4393 4391 4391
query15 213 203 180 180
query16 1044 465 453 453
query17 1153 780 645 645
query18 2742 485 366 366
query19 238 226 173 173
query20 148 176 129 129
query21 218 138 117 117
query22 13639 13544 13382 13382
query23 17166 16341 15864 15864
query23_1 16135 16092 16152 16092
query24 7487 1741 1378 1378
query24_1 1344 1349 1339 1339
query25 591 499 435 435
query26 1306 310 168 168
query27 2696 579 351 351
query28 4383 1947 1937 1937
query29 984 638 510 510
query30 308 247 203 203
query31 1115 1076 946 946
query32 87 76 73 73
query33 544 362 290 290
query34 1186 1130 633 633
query35 752 775 668 668
query36 1323 1299 1186 1186
query37 153 105 88 88
query38 3202 3152 3051 3051
query39 932 929 910 910
query39_1 875 892 871 871
query40 242 159 136 136
query41 66 67 61 61
query42 110 114 109 109
query43 322 333 280 280
query44
query45 210 203 193 193
query46 1072 1210 736 736
query47 2265 2345 2258 2258
query48 380 409 307 307
query49 646 531 422 422
query50 690 292 219 219
query51 4239 4300 4229 4229
query52 109 106 95 95
query53 248 276 207 207
query54 316 269 265 265
query55 97 89 83 83
query56 295 295 309 295
query57 1404 1369 1295 1295
query58 307 295 279 279
query59 1577 1627 1393 1393
query60 359 346 328 328
query61 180 179 179 179
query62 678 639 552 552
query63 243 205 210 205
query64 2451 856 710 710
query65
query66 1720 534 414 414
query67 30180 29954 29787 29787
query68
query69 481 353 321 321
query70 1017 958 1011 958
query71 307 289 276 276
query72 3016 2760 2398 2398
query73 876 771 426 426
query74 5060 4861 4691 4691
query75 2756 2658 2334 2334
query76 2306 1134 786 786
query77 413 415 356 356
query78 12992 12944 12290 12290
query79 1530 963 750 750
query80 1383 575 487 487
query81 514 277 248 248
query82 1228 159 125 125
query83 361 275 247 247
query84 261 138 112 112
query85 951 523 471 471
query86 469 346 333 333
query87 3431 3340 3190 3190
query88 3548 2668 2641 2641
query89 452 379 339 339
query90 1918 191 184 184
query91 174 170 137 137
query92 82 79 74 74
query93 943 958 583 583
query94 719 340 303 303
query95 665 394 436 394
query96 1096 740 322 322
query97 2699 2686 2551 2551
query98 239 229 236 229
query99 1164 1126 970 970
Total cold run time: 255027 ms
Total hot run time: 170549 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441477205
# BE UT Coverage Report
Increment line coverage `49.17% (914/1859)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/9084f82a0ae8c8ca055f19549ac56d6fb71a510c_9084f82a0ae8c8ca055f19549ac56d6fb71a510c/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/9084f82a0ae8c8ca055f19549ac56d6fb71a510c_9084f82a0ae8c8ca055f19549ac56d6fb71a510c/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.58% (20677/38588) |
| Line Coverage | 37.25% (195828/525772) |
| Region Coverage | 33.69% (153330/455166) |
| Branch Coverage | 34.67% (66969/193169) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3234983121
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,549 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <unordered_set>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr > end || size > static_cast<size_t>(end - ptr)) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+Status require_available_entries(const uint8_t* ptr, const uint8_t* end, uint64_t entries,
+ size_t entry_size, std::string_view context) {
+ if (entries > std::numeric_limits<size_t>::max() / entry_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return require_available(ptr, end, static_cast<size_t>(entries) * entry_size, context);
+}
+
+Status validate_array_field_offsets(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::string_view context) {
+ if (field_offsets.empty() || field_offsets.front() != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ for (size_t i = 0; i < field_offsets.size(); ++i) {
+ if (field_offsets[i] > total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offset {}", context,
+ field_offsets[i]);
+ }
+ if (i > 0 && field_offsets[i] < field_offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT {} field offsets", context);
+ }
+ }
+ return Status::OK();
+}
+
+Status compute_object_field_ends(const std::vector<uint64_t>& field_offsets, uint64_t total_size,
+ std::vector<uint64_t>* field_ends) {
+ if (field_offsets.empty()) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ size_t num_elements = field_offsets.size() - 1;
+ if (num_elements == 0) {
+ if (total_size != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ return Status::OK();
+ }
+
+ std::vector<std::pair<uint64_t, size_t>> physical_offsets;
+ physical_offsets.reserve(num_elements);
+ for (size_t i = 0; i < num_elements; ++i) {
+ if (field_offsets[i] >= total_size) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offset {}",
+ field_offsets[i]);
+ }
+ physical_offsets.emplace_back(field_offsets[i], i);
+ }
+ std::sort(physical_offsets.begin(), physical_offsets.end());
+ if (physical_offsets.front().first != 0) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+
+ field_ends->assign(num_elements, 0);
+ for (size_t i = 0; i < physical_offsets.size(); ++i) {
+ if (i > 0 && physical_offsets[i].first == physical_offsets[i - 1].first) {
+ return Status::Corruption("Invalid Parquet VARIANT object field offsets");
+ }
+ uint64_t child_end =
+ i + 1 < physical_offsets.size() ? physical_offsets[i + 1].first : total_size;
+ (*field_ends)[physical_offsets[i].second] = child_end;
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+void append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ json->append("null");
+ return;
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ RETURN_IF_ERROR(require_available_entries(ptr, end, dictionary_size + 1, offset_size,
+ "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ std::unordered_set<std::string> dictionary_entries;
+ dictionary_entries.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ std::string entry(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
+ if (!dictionary_entries.insert(entry).second) {
+ return Status::Corruption(
+ "Invalid Parquet VARIANT metadata duplicate dictionary key {}", entry);
+ }
+ result->dictionary.emplace_back(std::move(entry));
+ }
+ return Status::OK();
+}
+
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next) {
+ switch (primitive_header) {
+ case 0:
+ json->append("null");
+ *next = ptr;
+ return Status::OK();
+ case 1:
+ json->append("true");
+ *next = ptr;
+ return Status::OK();
+ case 2:
+ json->append("false");
+ *next = ptr;
+ return Status::OK();
+ case 3:
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "int8 value"));
+ json->append(std::to_string(static_cast<int8_t>(*ptr)));
+ *next = ptr + 1;
+ return Status::OK();
+ case 4:
+ RETURN_IF_ERROR(require_available(ptr, end, 2, "int16 value"));
+ json->append(std::to_string(read_signed_le(ptr, 2)));
+ *next = ptr + 2;
+ return Status::OK();
+ case 5:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "int32 value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 6:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "int64 value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 7: {
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "double value"));
+ uint64_t bits = read_unsigned_le(ptr, 8);
+ double value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 8;
+ return Status::OK();
+ }
+ case 8:
+ case 9:
+ case 10: {
+ int value_size = primitive_header == 8 ? 4 : primitive_header == 9 ? 8 : 16;
+ RETURN_IF_ERROR(require_available(ptr, end, 1 + value_size, "decimal value"));
+ int scale = static_cast<int8_t>(*ptr++);
+ __int128 unscaled = 0;
+ if (value_size == 16) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ if ((ptr[15] & 0x80) != 0) {
+ unscaled = -static_cast<__int128>((~unsigned_value) + 1);
Review Comment:
For decimal128, the minimum two's-complement value has magnitude `2^127`. In that case `(~unsigned_value) + 1` is `2^127`, casting it to signed `__int128` and then negating overflows/has implementation-defined behavior, so a valid Parquet VARIANT decimal128 payload can decode incorrectly. Please decode the signed 128-bit value without negating the unrepresentable minimum magnitude, or reject that payload explicitly with coverage.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4435733369
<details>
<summary>TPC-DS: <b>Total hot run time: 170914 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 7c992131eb1a6f8bbd285fd83cecec07b53e5bfe, data reload: false
query5 4317 684 506 506
query6 327 224 204 204
query7 4225 587 306 306
query8 319 235 223 223
query9 8856 4003 4012 4003
query10 453 379 301 301
query11 5808 2496 2196 2196
query12 183 133 128 128
query13 1289 632 438 438
query14 6767 5507 5072 5072
query14_1 4398 4411 4382 4382
query15 214 206 182 182
query16 1007 445 434 434
query17 1135 779 613 613
query18 2754 516 362 362
query19 224 207 162 162
query20 141 135 132 132
query21 215 143 116 116
query22 13654 13542 13374 13374
query23 17311 16450 16146 16146
query23_1 16205 16209 16137 16137
query24 7409 1861 1367 1367
query24_1 1405 1360 1385 1360
query25 607 542 485 485
query26 1303 347 183 183
query27 2715 623 355 355
query28 4455 1995 1972 1972
query29 1050 697 556 556
query30 309 256 194 194
query31 1149 1106 955 955
query32 86 75 78 75
query33 554 381 305 305
query34 1173 1191 649 649
query35 803 830 684 684
query36 1308 1319 1181 1181
query37 158 113 95 95
query38 3260 3153 3062 3062
query39 930 918 890 890
query39_1 864 865 885 865
query40 247 170 148 148
query41 72 70 105 70
query42 109 111 111 111
query43 329 335 281 281
query44
query45 220 202 192 192
query46 1112 1275 738 738
query47 2257 2312 2172 2172
query48 394 394 289 289
query49 634 564 424 424
query50 727 315 222 222
query51 4401 4280 4184 4184
query52 114 115 101 101
query53 259 301 203 203
query54 311 281 255 255
query55 102 90 84 84
query56 312 321 304 304
query57 1379 1363 1269 1269
query58 310 275 264 264
query59 1589 1672 1509 1509
query60 343 355 336 336
query61 155 159 162 159
query62 671 636 566 566
query63 239 202 207 202
query64 2347 843 685 685
query65
query66 1690 560 404 404
query67 30154 30050 29886 29886
query68
query69 469 349 316 316
query70 1040 987 962 962
query71 313 287 310 287
query72 2992 2788 2409 2409
query73 844 760 396 396
query74 5133 4934 4726 4726
query75 3006 2783 2354 2354
query76 2322 1215 810 810
query77 434 443 350 350
query78 13066 12977 12428 12428
query79 1528 977 729 729
query80 746 604 508 508
query81 471 290 244 244
query82 1273 163 124 124
query83 359 284 256 256
query84 269 144 118 118
query85 868 555 481 481
query86 398 343 328 328
query87 3439 3389 3201 3201
query88 3597 2672 2638 2638
query89 461 385 342 342
query90 1945 192 179 179
query91 183 170 141 141
query92 79 83 77 77
query93 990 1019 559 559
query94 535 356 310 310
query95 678 482 349 349
query96 1051 771 342 342
query97 2707 2692 2560 2560
query98 244 233 233 233
query99 1121 1107 988 988
Total cold run time: 255139 ms
Total hot run time: 170914 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4435682648
<details>
<summary>TPC-H: <b>Total hot run time: 29867 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 7c992131eb1a6f8bbd285fd83cecec07b53e5bfe, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17607 3885 3867 3867
q2 q3 10710 879 627 627
q4 4670 473 359 359
q5 7630 1332 1145 1145
q6 240 168 141 141
q7 972 941 751 751
q8 10024 1458 1301 1301
q9 6321 5409 5333 5333
q10 6364 2071 1796 1796
q11 475 277 264 264
q12 688 416 297 297
q13 18285 3378 2772 2772
q14 297 288 261 261
q15 q16 911 851 789 789
q17 1019 1038 849 849
q18 6501 5742 5636 5636
q19 1171 1189 1121 1121
q20 510 394 309 309
q21 4916 2427 1931 1931
q22 476 394 318 318
Total cold run time: 99787 ms
Total hot run time: 29867 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4666 4571 4668 4571
q2 q3 4641 4805 4218 4218
q4 2149 2194 1428 1428
q5 5047 5054 5276 5054
q6 202 174 136 136
q7 2038 1802 1608 1608
q8 3369 3162 3124 3124
q9 8564 8787 8490 8490
q10 4503 4486 4317 4317
q11 613 436 399 399
q12 685 744 558 558
q13 3229 3545 2948 2948
q14 313 315 281 281
q15 q16 770 786 694 694
q17 1384 1316 1249 1249
q18 8061 7031 7163 7031
q19 1176 1196 1139 1139
q20 2242 2219 1953 1953
q21 6277 5521 4929 4929
q22 536 468 401 401
Total cold run time: 60465 ms
Total hot run time: 54528 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3231368634
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1157,393 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
+ case TYPE_FLOAT: {
+ const auto value = field.get<TYPE_FLOAT>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_DOUBLE: {
+ const auto value = field.get<TYPE_DOUBLE>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_TIMEV2:
+ json->append(std::to_string(static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>()))));
+ return Status::OK();
+ case TYPE_DATE:
+ json->append(std::to_string(variant_date_value(field.get<TYPE_DATE>())));
+ return Status::OK();
+ case TYPE_DATETIME:
+ json->append(std::to_string(variant_datetime_value(field.get<TYPE_DATETIME>())));
+ return Status::OK();
+ case TYPE_DATEV2:
+ json->append(std::to_string(variant_date_value(field.get<TYPE_DATEV2>())));
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ json->append(std::to_string(variant_datetime_value(field.get<TYPE_DATETIMEV2>())));
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ json->append(std::to_string(variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>())));
+ return Status::OK();
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR: {
+ std::string value = field.to_debug_string(type->get_scale());
+ append_json_string(value, json);
+ return Status::OK();
+ }
+ case TYPE_VARBINARY:
+ return Status::NotSupported("Parquet VARIANT binary typed_value is not supported");
+ case TYPE_ARRAY:
+ return _array_field_to_json(field_schema, field,
+ assert_cast<const DataTypeArray*>(type.get()), json);
+ case TYPE_STRUCT:
+ return _struct_field_to_json(field_schema, field, json);
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+Status VariantColumnReader::_array_field_to_json(const FieldSchema& field_schema,
+ const Field& field,
+ const DataTypeArray* array_type,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_ARRAY>();
+ const FieldSchema* element_schema =
+ field_schema.children.empty() ? nullptr : field_schema.children.data();
+ FieldSchema synthetic_element;
+ if (element_schema == nullptr) {
+ synthetic_element.data_type = array_type->get_nested_type();
+ element_schema = &synthetic_element;
+ }
+ json->push_back('[');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(
+ _field_to_json(*element_schema, values[i], &element_json, &element_present));
+ json->append(element_present ? element_json : "null");
+ }
+ json->push_back(']');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_struct_field_to_json(const FieldSchema& field_schema,
+ const Field& field, std::string* json) const {
+ const auto& values = field.get<TYPE_STRUCT>();
+ json->push_back('{');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ const FieldSchema& child_schema = field_schema.children[i];
+ append_json_string(child_schema.name, json);
+ json->push_back(':');
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(_field_to_json(child_schema, values[i], &child_json, &child_present));
+ json->append(child_present ? child_json : "null");
+ }
+ json->push_back('}');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_typed_value_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ *present = true;
+ return _typed_struct_to_json(typed_value_field, field, metadata, json);
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ *present = true;
+ return _typed_array_to_json(typed_value_field, field, metadata, json);
+ }
+ return _field_to_json(typed_value_field, field, json, present);
+}
+
+Status VariantColumnReader::_shredded_field_to_json(const FieldSchema& field_schema,
+ const Field& field, const std::string& metadata,
+ std::string* json, bool* present) const {
+ if (is_variant_wrapper_field(field_schema)) {
+ return _variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ return _typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+Status VariantColumnReader::_typed_struct_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_STRUCT>();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(_shredded_field_to_json(typed_value_field.children[i], values[i], metadata,
+ &child_json, &child_present));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ first = false;
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ }
+ json->push_back('}');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_typed_array_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_ARRAY>();
+ const FieldSchema* element_schema =
+ typed_value_field.children.empty() ? nullptr : typed_value_field.children.data();
+ json->push_back('[');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ if (element_schema != nullptr) {
+ RETURN_IF_ERROR(_shredded_field_to_json(*element_schema, values[i], metadata,
+ &element_json, &element_present));
+ } else {
+ RETURN_IF_ERROR(
+ _field_to_json(typed_value_field, values[i], &element_json, &element_present));
+ }
+ json->append(element_present ? element_json : "null");
+ }
+ json->push_back(']');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& values = field.get<TYPE_STRUCT>();
+ int metadata_idx = find_child_idx(variant_field, "metadata");
+ int value_idx = find_child_idx(variant_field, "value");
+ int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(_get_binary_field(values[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ std::string value_json;
+ bool value_present = false;
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(_get_binary_field(values[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ }
+ }
+
+ std::string typed_json;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(_typed_value_to_json(variant_field.children[typed_value_idx],
+ values[typed_value_idx], metadata, &typed_json,
+ &typed_present));
+ }
+
+ if (typed_present) {
+ if (value_present) {
+ RETURN_IF_ERROR(merge_json_objects(value_json, typed_json, json));
+ } else {
+ *json = std::move(typed_json);
+ }
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+ *present = false;
+ return Status::OK();
+}
+
+Status VariantColumnReader::read_column_data(
+ ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
+ (void)root_node;
+ if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_VARIANT) {
+ return Status::Corruption(
+ "Wrong data type for column '{}', expected Variant type, actual type: {}.",
+ _field_schema->name, type->get_name());
+ }
+
+ ColumnPtr struct_column = _variant_struct_type->create_column();
+ const size_t old_struct_rows = struct_column->size();
+ auto const_node = TableSchemaChangeHelper::ConstNode::get_instance();
+ RETURN_IF_ERROR(_struct_reader->read_column_data(struct_column, _variant_struct_type,
+ const_node, filter_map, batch_size, read_rows,
+ eof, is_dict_filter, real_column_size));
+
+ const size_t new_struct_rows = struct_column->size() - old_struct_rows;
+ if (new_struct_rows == 0) {
+ return Status::OK();
+ }
+
+ MutableColumnPtr variant_column_ptr;
+ NullMap* null_map_ptr = nullptr;
+ auto mutable_column = doris_column->assume_mutable();
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = assert_cast<ColumnNullable*>(mutable_column.get());
+ variant_column_ptr = nullable_column->get_nested_column_ptr();
+ null_map_ptr = &nullable_column->get_null_map_data();
+ } else {
+ if (_field_schema->data_type->is_nullable()) {
+ return Status::Corruption("Not nullable column has null values in parquet file");
+ }
+ variant_column_ptr = std::move(mutable_column);
+ }
+ auto* variant_column = assert_cast<ColumnVariant*>(variant_column_ptr.get());
+
+ ParseConfig parse_config;
+ for (size_t i = old_struct_rows; i < struct_column->size(); ++i) {
+ std::string json;
+ bool present = false;
+ RETURN_IF_ERROR(
+ _variant_to_json(*_field_schema, (*struct_column)[i], nullptr, &json, &present));
+ if (!present) {
+ if (null_map_ptr != nullptr) {
+ variant_column->insert_default();
+ null_map_ptr->push_back(1);
+ continue;
+ }
+ json = "null";
+ }
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(variant_util::parse_json_to_variant(*variant_column, json_ref,
Review Comment:
能不能用JSONB来表达,不走parse, 再decode的时候变成JSONB, JSONB是ColumnVariant里的节点
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1157,393 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
+ case TYPE_FLOAT: {
+ const auto value = field.get<TYPE_FLOAT>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_DOUBLE: {
+ const auto value = field.get<TYPE_DOUBLE>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_TIMEV2:
+ json->append(std::to_string(static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>()))));
+ return Status::OK();
+ case TYPE_DATE:
+ json->append(std::to_string(variant_date_value(field.get<TYPE_DATE>())));
+ return Status::OK();
+ case TYPE_DATETIME:
+ json->append(std::to_string(variant_datetime_value(field.get<TYPE_DATETIME>())));
+ return Status::OK();
+ case TYPE_DATEV2:
+ json->append(std::to_string(variant_date_value(field.get<TYPE_DATEV2>())));
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ json->append(std::to_string(variant_datetime_value(field.get<TYPE_DATETIMEV2>())));
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ json->append(std::to_string(variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>())));
+ return Status::OK();
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR: {
+ std::string value = field.to_debug_string(type->get_scale());
+ append_json_string(value, json);
+ return Status::OK();
+ }
+ case TYPE_VARBINARY:
+ return Status::NotSupported("Parquet VARIANT binary typed_value is not supported");
+ case TYPE_ARRAY:
+ return _array_field_to_json(field_schema, field,
+ assert_cast<const DataTypeArray*>(type.get()), json);
+ case TYPE_STRUCT:
+ return _struct_field_to_json(field_schema, field, json);
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+Status VariantColumnReader::_array_field_to_json(const FieldSchema& field_schema,
+ const Field& field,
+ const DataTypeArray* array_type,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_ARRAY>();
+ const FieldSchema* element_schema =
+ field_schema.children.empty() ? nullptr : field_schema.children.data();
+ FieldSchema synthetic_element;
+ if (element_schema == nullptr) {
+ synthetic_element.data_type = array_type->get_nested_type();
+ element_schema = &synthetic_element;
+ }
+ json->push_back('[');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(
+ _field_to_json(*element_schema, values[i], &element_json, &element_present));
+ json->append(element_present ? element_json : "null");
+ }
+ json->push_back(']');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_struct_field_to_json(const FieldSchema& field_schema,
+ const Field& field, std::string* json) const {
+ const auto& values = field.get<TYPE_STRUCT>();
+ json->push_back('{');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ const FieldSchema& child_schema = field_schema.children[i];
+ append_json_string(child_schema.name, json);
+ json->push_back(':');
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(_field_to_json(child_schema, values[i], &child_json, &child_present));
+ json->append(child_present ? child_json : "null");
+ }
+ json->push_back('}');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_typed_value_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ *present = true;
+ return _typed_struct_to_json(typed_value_field, field, metadata, json);
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ *present = true;
+ return _typed_array_to_json(typed_value_field, field, metadata, json);
+ }
+ return _field_to_json(typed_value_field, field, json, present);
+}
+
+Status VariantColumnReader::_shredded_field_to_json(const FieldSchema& field_schema,
+ const Field& field, const std::string& metadata,
+ std::string* json, bool* present) const {
+ if (is_variant_wrapper_field(field_schema)) {
+ return _variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ return _typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+Status VariantColumnReader::_typed_struct_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_STRUCT>();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(_shredded_field_to_json(typed_value_field.children[i], values[i], metadata,
+ &child_json, &child_present));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ first = false;
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ }
+ json->push_back('}');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_typed_array_to_json(const FieldSchema& typed_value_field,
+ const Field& field, const std::string& metadata,
+ std::string* json) const {
+ const auto& values = field.get<TYPE_ARRAY>();
+ const FieldSchema* element_schema =
+ typed_value_field.children.empty() ? nullptr : typed_value_field.children.data();
+ json->push_back('[');
+ for (int i = 0; i < values.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ if (element_schema != nullptr) {
+ RETURN_IF_ERROR(_shredded_field_to_json(*element_schema, values[i], metadata,
+ &element_json, &element_present));
+ } else {
+ RETURN_IF_ERROR(
+ _field_to_json(typed_value_field, values[i], &element_json, &element_present));
+ }
+ json->append(element_present ? element_json : "null");
+ }
+ json->push_back(']');
+ return Status::OK();
+}
+
+Status VariantColumnReader::_variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& values = field.get<TYPE_STRUCT>();
+ int metadata_idx = find_child_idx(variant_field, "metadata");
+ int value_idx = find_child_idx(variant_field, "value");
+ int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(_get_binary_field(values[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ std::string value_json;
+ bool value_present = false;
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(_get_binary_field(values[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ }
+ }
+
+ std::string typed_json;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(_typed_value_to_json(variant_field.children[typed_value_idx],
+ values[typed_value_idx], metadata, &typed_json,
+ &typed_present));
+ }
+
+ if (typed_present) {
+ if (value_present) {
+ RETURN_IF_ERROR(merge_json_objects(value_json, typed_json, json));
+ } else {
+ *json = std::move(typed_json);
+ }
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+ *present = false;
+ return Status::OK();
+}
+
+Status VariantColumnReader::read_column_data(
+ ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
+ (void)root_node;
+ if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_VARIANT) {
+ return Status::Corruption(
+ "Wrong data type for column '{}', expected Variant type, actual type: {}.",
+ _field_schema->name, type->get_name());
+ }
+
+ ColumnPtr struct_column = _variant_struct_type->create_column();
+ const size_t old_struct_rows = struct_column->size();
+ auto const_node = TableSchemaChangeHelper::ConstNode::get_instance();
+ RETURN_IF_ERROR(_struct_reader->read_column_data(struct_column, _variant_struct_type,
+ const_node, filter_map, batch_size, read_rows,
+ eof, is_dict_filter, real_column_size));
+
+ const size_t new_struct_rows = struct_column->size() - old_struct_rows;
+ if (new_struct_rows == 0) {
+ return Status::OK();
+ }
+
+ MutableColumnPtr variant_column_ptr;
+ NullMap* null_map_ptr = nullptr;
+ auto mutable_column = doris_column->assume_mutable();
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = assert_cast<ColumnNullable*>(mutable_column.get());
+ variant_column_ptr = nullable_column->get_nested_column_ptr();
+ null_map_ptr = &nullable_column->get_null_map_data();
+ } else {
+ if (_field_schema->data_type->is_nullable()) {
+ return Status::Corruption("Not nullable column has null values in parquet file");
+ }
+ variant_column_ptr = std::move(mutable_column);
+ }
+ auto* variant_column = assert_cast<ColumnVariant*>(variant_column_ptr.get());
+
+ ParseConfig parse_config;
+ for (size_t i = old_struct_rows; i < struct_column->size(); ++i) {
+ std::string json;
+ bool present = false;
+ RETURN_IF_ERROR(
+ _variant_to_json(*_field_schema, (*struct_column)[i], nullptr, &json, &present));
+ if (!present) {
+ if (null_map_ptr != nullptr) {
+ variant_column->insert_default();
+ null_map_ptr->push_back(1);
+ continue;
+ }
+ json = "null";
+ }
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(variant_util::parse_json_to_variant(*variant_column, json_ref,
Review Comment:
能不能用JSONB来表达,不走parse, 再decode的时候变成JSONB, JSONB是ColumnVariant里的节点
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437253910
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437254897
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3231622050
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +114,410 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ bool has_typed_value = false;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ has_typed_value = true;
+ continue;
+ }
+ return false;
+ }
+ return has_typed_value || (has_metadata && has_value);
+}
+
+static const IColumn& remove_nullable_column(const IColumn& column) {
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(&column)) {
+ return nullable->get_nested_column();
+ }
+ return column;
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema) {
+ if (is_variant_wrapper_field(field_schema)) {
+ const int value_idx = find_child_idx(field_schema, "value");
Review Comment:
This wrapper detection is used by the typed-only direct-read fast path without knowing whether `field_schema` is the top-level `typed_value` object-field container or an actual shredded-field wrapper. A valid VARIANT object can have a key named `typed_value`; for a typed-only schema like `v.typed_value.typed_value` where that key is an `INT`, `can_direct_read_typed_value(_field_schema->children[typed_value_idx])` treats the top-level `typed_value` struct itself as a wrapper, unwraps the user field, and `append_direct_typed_column_to_batch()` writes the primitive at the root path instead of path `typed_value`. That changes `v` from an object (`{"typed_value": 1}`) into a scalar and makes `v['typed_value']` wrong. Please make wrapper detection context-aware (do not unwrap the top-level typed-value object container just because a user key collides with the structural name) and add coverage for a VARIANT key named `typed_value` (and ideally `value`) in typed-only shredding.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437758910
# BE UT Coverage Report
Increment line coverage `6.37% (91/1428)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/1c9139f6149a046da1ff1bbc14553bb23b8ad35f_1c9139f6149a046da1ff1bbc14553bb23b8ad35f/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/1c9139f6149a046da1ff1bbc14553bb23b8ad35f_1c9139f6149a046da1ff1bbc14553bb23b8ad35f/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.47% (20638/38597) |
| Line Coverage | 37.11% (195066/525587) |
| Region Coverage | 33.52% (152480/454833) |
| Branch Coverage | 34.47% (66507/192935) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437836829
<details>
<summary>TPC-DS: <b>Total hot run time: 171692 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit bcda6244eb29152d7533075da24e9c6b6dc1990f, data reload: false
query5 4330 680 525 525
query6 336 219 201 201
query7 4227 565 308 308
query8 328 229 219 219
query9 8826 4060 4073 4060
query10 451 355 295 295
query11 5799 2404 2206 2206
query12 188 132 127 127
query13 1373 632 438 438
query14 6084 5435 5109 5109
query14_1 4456 4430 4382 4382
query15 216 208 187 187
query16 1018 479 453 453
query17 1136 761 641 641
query18 2555 493 372 372
query19 221 212 169 169
query20 150 136 135 135
query21 218 145 124 124
query22 13764 13988 14634 13988
query23 17424 16441 16243 16243
query23_1 16189 16249 16360 16249
query24 7403 1742 1348 1348
query24_1 1332 1313 1334 1313
query25 563 488 423 423
query26 1300 308 174 174
query27 2693 616 327 327
query28 4378 1972 1988 1972
query29 969 625 510 510
query30 299 226 195 195
query31 1111 1065 945 945
query32 86 70 71 70
query33 588 337 279 279
query34 1162 1127 643 643
query35 780 763 668 668
query36 1311 1322 1129 1129
query37 149 104 90 90
query38 3211 3139 3069 3069
query39 923 912 879 879
query39_1 867 879 864 864
query40 232 153 133 133
query41 62 58 59 58
query42 107 105 106 105
query43 333 324 295 295
query44
query45 215 198 200 198
query46 1099 1215 720 720
query47 2331 2266 2116 2116
query48 383 417 285 285
query49 634 526 419 419
query50 723 294 227 227
query51 4314 4267 4182 4182
query52 104 105 93 93
query53 250 283 208 208
query54 301 281 262 262
query55 90 88 82 82
query56 293 306 301 301
query57 1399 1370 1261 1261
query58 299 277 270 270
query59 1588 1618 1392 1392
query60 335 328 324 324
query61 157 167 155 155
query62 657 625 561 561
query63 248 202 201 201
query64 2406 856 675 675
query65
query66 1738 500 388 388
query67 29906 29902 29792 29792
query68
query69 476 345 308 308
query70 1039 1014 992 992
query71 302 279 272 272
query72 2979 2736 2468 2468
query73 804 717 451 451
query74 5135 4899 4705 4705
query75 2780 2645 2330 2330
query76 2267 1124 739 739
query77 421 419 355 355
query78 12974 13069 12352 12352
query79 1527 968 744 744
query80 1372 576 476 476
query81 519 278 242 242
query82 995 161 126 126
query83 356 280 254 254
query84 262 142 113 113
query85 950 495 438 438
query86 452 346 329 329
query87 3456 3371 3221 3221
query88 3526 2665 2669 2665
query89 441 379 338 338
query90 1938 184 189 184
query91 178 171 139 139
query92 82 78 74 74
query93 1081 931 553 553
query94 730 335 302 302
query95 693 371 346 346
query96 1004 758 336 336
query97 2702 2691 2585 2585
query98 242 232 246 232
query99 1088 1089 971 971
Total cold run time: 254444 ms
Total hot run time: 171692 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4439047075
<details>
<summary>TPC-DS: <b>Total hot run time: 171455 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 448355b9ec5afaa1afb2b0910fd3a6825e1007cf, data reload: false
query5 4346 644 531 531
query6 336 240 203 203
query7 4306 562 307 307
query8 329 241 228 228
query9 8810 4006 3992 3992
query10 460 343 303 303
query11 5815 2362 2238 2238
query12 182 130 126 126
query13 1321 601 425 425
query14 5976 5372 5030 5030
query14_1 4369 4350 4352 4350
query15 213 206 185 185
query16 1013 495 445 445
query17 996 772 660 660
query18 2461 503 383 383
query19 226 242 157 157
query20 140 134 132 132
query21 218 135 120 120
query22 13549 13919 14628 13919
query23 17399 16588 16288 16288
query23_1 16223 16286 16267 16267
query24 7435 1735 1330 1330
query24_1 1367 1335 1330 1330
query25 589 484 426 426
query26 1297 322 169 169
query27 2734 606 340 340
query28 4435 1963 1923 1923
query29 1045 618 514 514
query30 303 233 202 202
query31 1098 1058 921 921
query32 86 71 70 70
query33 552 342 288 288
query34 1180 1129 628 628
query35 766 765 677 677
query36 1369 1291 1183 1183
query37 148 100 80 80
query38 3185 3118 3067 3067
query39 919 930 885 885
query39_1 862 858 884 858
query40 236 152 135 135
query41 63 61 59 59
query42 112 103 108 103
query43 325 326 287 287
query44
query45 208 191 195 191
query46 1071 1178 722 722
query47 2338 2338 2220 2220
query48 398 443 310 310
query49 642 541 424 424
query50 684 280 221 221
query51 4235 4272 4183 4183
query52 104 104 94 94
query53 254 265 196 196
query54 320 306 247 247
query55 92 87 85 85
query56 296 300 311 300
query57 1427 1423 1364 1364
query58 305 275 263 263
query59 1562 1569 1412 1412
query60 343 332 323 323
query61 161 157 160 157
query62 669 619 562 562
query63 246 211 202 202
query64 2462 821 674 674
query65
query66 1759 507 393 393
query67 30097 29941 29901 29901
query68
query69 470 340 299 299
query70 1019 1003 935 935
query71 305 274 271 271
query72 2940 2759 2652 2652
query73 803 745 430 430
query74 5062 4935 4751 4751
query75 2807 2669 2350 2350
query76 2324 1124 760 760
query77 445 436 375 375
query78 12984 12961 12319 12319
query79 1474 1035 730 730
query80 707 610 510 510
query81 451 287 237 237
query82 1335 162 128 128
query83 374 283 259 259
query84 317 146 119 119
query85 941 518 456 456
query86 408 315 311 311
query87 3379 3348 3233 3233
query88 3477 2633 2616 2616
query89 432 375 338 338
query90 1873 185 171 171
query91 180 176 139 139
query92 74 77 74 74
query93 937 947 550 550
query94 550 336 279 279
query95 647 479 338 338
query96 1039 757 331 331
query97 2709 2737 2588 2588
query98 241 226 237 226
query99 1115 1110 996 996
Total cold run time: 253238 ms
Total hot run time: 171455 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4439000297
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3229423008
##########
be/src/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp:
##########
@@ -18,21 +18,173 @@
#include "format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <iostream>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_name(const FieldSchema& field_schema, std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+bool extract_nested_column_ids_by_name(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+bool extract_variant_nested_column_ids(const FieldSchema& variant_field,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_name(variant_field, "typed_value");
+ bool has_child_columns = false;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ has_child_columns = true;
+ continue;
+ }
+
+ add_variant_metadata(variant_field, column_ids);
+ bool found_typed_path = false;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ found_typed_path = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ found_typed_path = extract_nested_column_ids_by_name(*typed_child, child_paths,
+ column_ids);
+ }
+ if (found_typed_path) {
+ column_ids.insert(typed_value->get_column_id());
+ }
+ }
+ }
+
+ if (!found_typed_path) {
+ add_variant_value(variant_field, column_ids);
Review Comment:
Same answer as the Hive/local path. The helper keeps typed_value.<field>.value by adding the selected typed child range; it intentionally does not select top-level value for a shredded field, because the Parquet Variant Shredding spec says top-level value must contain only non-shredded object fields and shredded field residuals live in typed_value.<field>.value. The regression observable now asserts both metric leaves and excludes v.value/name siblings. Source: https://parquet.apache.org/docs/file-format/types/variantshredding/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434824417
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3238636082
##########
be/src/format/parquet/parquet_column_convert.h:
##########
@@ -175,6 +176,18 @@ struct ConvertParams {
second_mask = 1000000000;
scale_to_nano_factor = 1;
}
+ } else if (schema.__isset.logicalType && schema.logicalType.__isset.TIME) {
+ const auto& time_unit = schema.logicalType.TIME.unit;
Review Comment:
为什么需要这些改动,能不能去掉无关的, 再梳理一下逻辑
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466640933
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466730828
<details>
<summary>TPC-DS: <b>Total hot run time: 168645 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 81c4010234456ca39d939382b19f1af624aeec55, data reload: false
query5 4323 676 520 520
query6 330 214 197 197
query7 4237 569 290 290
query8 327 224 218 218
query9 8823 3978 3935 3935
query10 454 340 310 310
query11 5755 2328 2166 2166
query12 182 125 123 123
query13 1265 620 448 448
query14 5949 5343 5031 5031
query14_1 4347 4353 4332 4332
query15 222 203 184 184
query16 1003 482 444 444
query17 1159 746 642 642
query18 2679 500 362 362
query19 225 207 167 167
query20 139 131 133 131
query21 222 136 127 127
query22 13600 13545 13292 13292
query23 17289 16640 16387 16387
query23_1 16452 16489 16326 16326
query24 7423 1732 1271 1271
query24_1 1259 1293 1288 1288
query25 540 466 418 418
query26 1307 315 167 167
query27 2771 533 329 329
query28 4452 1951 1906 1906
query29 952 606 484 484
query30 307 238 198 198
query31 1118 1055 942 942
query32 88 71 70 70
query33 525 358 317 317
query34 1169 1101 634 634
query35 761 772 667 667
query36 1317 1353 1180 1180
query37 157 103 91 91
query38 3202 3109 2988 2988
query39 926 914 905 905
query39_1 871 881 874 874
query40 232 148 125 125
query41 66 64 61 61
query42 111 106 106 106
query43 320 322 283 283
query44
query45 213 197 194 194
query46 1080 1223 704 704
query47 2381 2382 2238 2238
query48 397 405 288 288
query49 638 480 407 407
query50 1055 340 255 255
query51 4272 4303 4232 4232
query52 104 104 96 96
query53 242 284 204 204
query54 311 271 251 251
query55 91 87 84 84
query56 307 311 312 311
query57 1425 1395 1300 1300
query58 313 275 271 271
query59 1540 1575 1363 1363
query60 324 326 309 309
query61 155 156 158 156
query62 669 647 564 564
query63 251 200 199 199
query64 2405 813 644 644
query65
query66 1670 479 360 360
query67 29949 29887 29161 29161
query68
query69 464 330 297 297
query70 1020 982 979 979
query71 307 267 269 267
query72 2982 2634 2550 2550
query73 835 776 448 448
query74 5038 4873 4744 4744
query75 2664 2564 2260 2260
query76 2274 1115 798 798
query77 396 418 331 331
query78 12072 12019 11603 11603
query79 1383 1040 755 755
query80 707 573 475 475
query81 451 277 248 248
query82 1379 161 125 125
query83 369 283 255 255
query84 293 151 113 113
query85 994 617 545 545
query86 389 340 314 314
query87 3364 3407 3176 3176
query88 3529 2660 2651 2651
query89 443 383 343 343
query90 1946 183 176 176
query91 194 213 140 140
query92 81 81 72 72
query93 1486 1526 954 954
query94 532 361 278 278
query95 668 458 351 351
query96 1019 819 322 322
query97 2690 2668 2572 2572
query98 235 230 234 230
query99 1117 1107 985 985
Total cold run time: 252897 ms
Total hot run time: 168645 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467114182
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467219239
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467352453
<details>
<summary>TPC-H: <b>Total hot run time: 31215 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 8468e1e4b59fc6dd19d32fdad2b2fbf0fbc402c4, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17767 3843 3848 3843
q2 q3 10771 1313 811 811
q4 4690 473 342 342
q5 7639 2263 2081 2081
q6 240 183 140 140
q7 913 767 627 627
q8 9342 1758 1607 1607
q9 5076 4832 4846 4832
q10 6373 2077 1814 1814
q11 450 278 251 251
q12 623 417 292 292
q13 18108 3350 2741 2741
q14 261 257 245 245
q15 q16 817 762 708 708
q17 946 958 906 906
q18 6898 5910 5627 5627
q19 1312 1298 1117 1117
q20 515 402 263 263
q21 6384 2849 2641 2641
q22 450 390 327 327
Total cold run time: 99575 ms
Total hot run time: 31215 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4826 4487 4580 4487
q2 q3 4811 5264 4617 4617
q4 2127 2196 1409 1409
q5 4720 4783 4659 4659
q6 230 180 135 135
q7 1882 1766 1526 1526
q8 2351 2076 2110 2076
q9 7651 7578 7191 7191
q10 4475 4409 3972 3972
q11 529 387 353 353
q12 712 714 506 506
q13 3018 3367 2827 2827
q14 271 274 258 258
q15 q16 671 687 617 617
q17 1278 1245 1242 1242
q18 7129 6774 6701 6701
q19 1112 1113 1072 1072
q20 2222 2227 1913 1913
q21 5334 4603 4749 4603
q22 521 467 404 404
Total cold run time: 55870 ms
Total hot run time: 50568 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467408597
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253212940
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -115,7 +1667,7 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
size_t total_rows = row_group.num_rows;
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
Review Comment:
The factory still dispatches ARRAY/MAP/STRUCT using `field->data_type->get_primitive_type()` without unwrapping nullable, so the new VARIANT struct-reader path can still send a valid optional complex `typed_value` child into the scalar-reader branch. For example, a VARIANT schema with `optional group typed_value { optional int64 a; }` selected by `v['a']` reaches the top synthetic struct reader, then recurses into the `typed_value` child whose `data_type` is `Nullable(Struct(...))`; this skips the STRUCT branch and falls through with the group field's `physical_column_index == -1`. This is distinct from the earlier optional top-level VARIANT thread because the top synthetic struct is now non-nullable, but its optional complex children remain nullable. Please dispatch ARRAY/MAP/STRUCT on `remove_nullable(field->data_type)` as well, and add coverage for optional complex `typed_value` struct/list/map shards.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467687772
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253312414
##########
be/src/format/parquet/parquet_nested_column_utils.cpp:
##########
@@ -0,0 +1,529 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <cctype>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "core/data_type/data_type_nullable.h"
+#include "format/parquet/schema_desc.h"
+
+namespace doris {
+namespace {
+
+enum class NestedPathMode {
+ NAME,
+ FIELD_ID,
+};
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_variant_typed_child_by_key(const FieldSchema& field_schema,
+ std::string_view key, NestedPathMode mode) {
+ if (const auto* child = find_child_by_exact_name(field_schema, key)) {
+ return child;
+ }
+ if (mode == NestedPathMode::NAME) {
+ return nullptr;
+ }
+ for (const auto& child : field_schema.children) {
+ if (child.field_id >= 0 && key == std::to_string(child.field_id)) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+using VariantPathMap = std::unordered_map<std::string, std::vector<std::vector<std::string>>>;
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
Review Comment:
This also classifies an ordinary typed-only user object whose only child key is named `typed_value` as a shredded wrapper. For example, a valid typed layout for `{'obj': {'typed_value': {'x': 1}}}` has `v.typed_value.obj.typed_value.x`; querying `v['obj']['typed_value']['x']` recurses into `obj`, this predicate treats `obj` itself as a wrapper, and `extract_shredded_variant_field_ids()` consumes the user `typed_value` as wrapper metadata instead of selecting `obj.typed_value.x`. With no residual `value` child in a typed-only file, the helper reports no selected typed leaf and the scan falls back to the wrong top-level residual path/missing data. Please avoid treating a lone complex child named `typed_value` as a wrapper unless the caller can prove it is an unannotated shredded-field layout, and add coverage for a user object field named `typed_value`.
##########
be/src/format/parquet/vparquet_reader.cpp:
##########
@@ -432,11 +440,111 @@ Status ParquetReader::on_before_init_reader(ReaderInitContext* ctx) {
RETURN_IF_ERROR(get_file_metadata_schema(&field_desc));
RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
ctx->tuple_descriptor, *field_desc, ctx->table_info_node));
+ auto column_id_result = _create_column_ids_by_name(field_desc, ctx->tuple_descriptor);
+ ctx->column_ids = std::move(column_id_result.column_ids);
+ ctx->filter_column_ids = std::move(column_id_result.filter_column_ids);
}
return Status::OK();
}
Review Comment:
This mutates the `FieldDescriptor` returned from `FileMetaData::schema()`, but `_open_file()` can source `_file_metadata` from `_meta_cache`, so multiple readers/fragments scanning the same cached footer can call `assign_ids()` concurrently on the same schema object. `assign_ids()` writes `column_id`/`max_column_id` through the whole tree, so even though the values are deterministic this is still an unsynchronized write/write (and write/read, e.g. `_selected_leaf_column_paths()` / range generation) data race on cached metadata. Please assign these IDs before inserting the footer into the cache, guard the mutation, or use a per-reader copied schema instead of `const_cast`ing the shared cached schema.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java:
##########
@@ -618,6 +620,8 @@ public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type, bo
enableMappingVarbinary, enableMappingTimestampTz);
}
switch (type.typeId()) {
+ case VARIANT:
Review Comment:
Exposing VARIANT recursively inside Iceberg complex types needs matching Nereids pruning support. With a schema such as `struct<s: struct<v: variant>>` and `SELECT s.v['k'] FROM tbl`, access collection produces a nested path like `[s, v, k]` for the struct slot, but `NestedColumnPruning.DataTypeAccessTree.setAccessByPath()` has no `VariantType` case below a struct/map/array node. It throws `unsupported data type`, `rewriteRoot()` catches it, and the whole pruning/access-path rewrite is skipped, so nested Iceberg VARIANT subfield reads cannot use the shredded-column pruning this PR adds. Please either handle `VariantType` as an access-path leaf inside complex parents or do not expose nested Iceberg VARIANT until the pruning tree can represent it, with a regression for `struct<variant>` subfield access.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467834444
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468146500
<details>
<summary>TPC-H: <b>Total hot run time: 31168 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 788ad7a017b7fb12d358bc3f6112c03aea0d3be6, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17663 3829 3834 3829
q2 q3 10728 1446 830 830
q4 4704 477 346 346
q5 7663 2321 2141 2141
q6 241 179 145 145
q7 925 756 630 630
q8 9363 1797 1559 1559
q9 5589 4865 4885 4865
q10 6456 2076 1841 1841
q11 449 275 245 245
q12 634 423 300 300
q13 18116 3347 2792 2792
q14 260 254 237 237
q15 q16 821 761 712 712
q17 960 949 1029 949
q18 7051 5760 5498 5498
q19 1165 1356 1054 1054
q20 503 505 331 331
q21 6117 2979 2553 2553
q22 485 396 311 311
Total cold run time: 99893 ms
Total hot run time: 31168 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4841 4564 4526 4526
q2 q3 4880 5292 4567 4567
q4 2133 2452 1418 1418
q5 4751 4840 4634 4634
q6 226 187 137 137
q7 1849 1705 1595 1595
q8 2372 2067 2101 2067
q9 7733 7655 7172 7172
q10 4437 4386 3970 3970
q11 523 381 348 348
q12 702 715 505 505
q13 2974 3370 2842 2842
q14 265 271 248 248
q15 q16 671 697 610 610
q17 1259 1246 1224 1224
q18 7558 6902 6822 6822
q19 1150 1113 1124 1113
q20 2211 2222 1943 1943
q21 5305 4646 4509 4509
q22 508 460 409 409
Total cold run time: 56348 ms
Total hot run time: 50659 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468366373
<details>
<summary>TPC-DS: <b>Total hot run time: 169711 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 3853dff1ae1dd95a975ee64df294feab76fab9db, data reload: false
query5 4311 639 522 522
query6 353 220 200 200
query7 4245 545 308 308
query8 333 241 228 228
query9 8880 4008 3990 3990
query10 466 353 294 294
query11 5792 2394 2183 2183
query12 195 127 135 127
query13 1291 615 432 432
query14 5967 5331 5040 5040
query14_1 4391 4332 4367 4332
query15 206 205 183 183
query16 1011 461 434 434
query17 1159 738 618 618
query18 2727 485 370 370
query19 224 212 175 175
query20 147 135 128 128
query21 218 137 120 120
query22 13525 13536 13286 13286
query23 17260 16313 16054 16054
query23_1 16132 16137 16175 16137
query24 7435 1781 1298 1298
query24_1 1266 1308 1305 1305
query25 590 504 453 453
query26 1326 316 182 182
query27 2667 540 324 324
query28 4457 1961 1942 1942
query29 1030 641 520 520
query30 308 243 203 203
query31 1109 1066 948 948
query32 89 85 75 75
query33 563 360 300 300
query34 1184 1106 657 657
query35 765 777 738 738
query36 1359 1330 1191 1191
query37 149 103 92 92
query38 3186 3114 3048 3048
query39 930 918 913 913
query39_1 899 876 871 871
query40 227 144 132 132
query41 65 62 62 62
query42 120 109 109 109
query43 343 332 290 290
query44
query45 206 199 192 192
query46 1077 1216 727 727
query47 2284 2351 2123 2123
query48 387 426 298 298
query49 626 485 382 382
query50 1005 350 250 250
query51 4341 4283 4233 4233
query52 106 104 94 94
query53 251 285 199 199
query54 313 267 256 256
query55 93 86 84 84
query56 295 299 285 285
query57 1436 1417 1353 1353
query58 303 277 269 269
query59 1556 1679 1435 1435
query60 326 326 308 308
query61 161 157 154 154
query62 689 649 568 568
query63 244 203 210 203
query64 2261 799 621 621
query65
query66 1658 481 365 365
query67 30083 29981 29736 29736
query68
query69 469 339 303 303
query70 1066 985 988 985
query71 306 278 265 265
query72 2967 2716 2386 2386
query73 851 756 420 420
query74 5034 4942 4739 4739
query75 2642 2565 2240 2240
query76 2298 1141 771 771
query77 391 409 329 329
query78 12169 12062 11633 11633
query79 1601 1028 751 751
query80 1305 537 446 446
query81 537 278 233 233
query82 945 151 123 123
query83 335 270 257 257
query84 262 138 114 114
query85 904 532 439 439
query86 454 316 327 316
query87 3417 3363 3188 3188
query88 3554 2685 2653 2653
query89 449 384 338 338
query90 1878 174 184 174
query91 179 171 153 153
query92 78 79 73 73
query93 1658 1493 915 915
query94 707 373 323 323
query95 688 377 349 349
query96 1036 818 344 344
query97 2762 2681 2560 2560
query98 246 228 226 226
query99 1141 1103 991 991
Total cold run time: 253938 ms
Total hot run time: 169711 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468209702
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468516155
<details>
<summary>TPC-DS: <b>Total hot run time: 168565 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit b8256c78ced82ed5e103c6c26ddd5845bf179ee5, data reload: false
query5 4320 640 511 511
query6 339 220 207 207
query7 4275 584 322 322
query8 357 234 218 218
query9 8814 4019 3956 3956
query10 463 344 298 298
query11 5844 2402 2191 2191
query12 188 130 127 127
query13 1383 598 434 434
query14 5960 5347 5124 5124
query14_1 4361 4333 4344 4333
query15 210 205 189 189
query16 1009 407 399 399
query17 1169 739 609 609
query18 2511 514 362 362
query19 222 235 166 166
query20 137 135 125 125
query21 206 147 116 116
query22 13515 13652 13269 13269
query23 17281 16338 16028 16028
query23_1 16173 16247 16237 16237
query24 7443 1734 1309 1309
query24_1 1278 1286 1294 1286
query25 573 479 424 424
query26 1318 306 172 172
query27 2714 573 342 342
query28 4418 1964 1951 1951
query29 1018 625 503 503
query30 303 242 201 201
query31 1108 1064 936 936
query32 89 76 73 73
query33 525 356 286 286
query34 1165 1118 650 650
query35 756 786 658 658
query36 1334 1374 1182 1182
query37 153 101 94 94
query38 3211 3170 3050 3050
query39 923 909 903 903
query39_1 871 874 880 874
query40 229 149 124 124
query41 70 64 67 64
query42 117 112 112 112
query43 323 327 279 279
query44
query45 209 198 195 195
query46 1088 1266 734 734
query47 2249 2312 2172 2172
query48 367 443 305 305
query49 659 503 403 403
query50 969 368 250 250
query51 4380 4430 4327 4327
query52 109 110 98 98
query53 268 298 212 212
query54 335 281 267 267
query55 95 99 100 99
query56 307 313 332 313
query57 1457 1436 1342 1342
query58 311 284 277 277
query59 1639 1638 1405 1405
query60 320 326 316 316
query61 159 155 156 155
query62 679 626 538 538
query63 241 203 220 203
query64 2434 825 642 642
query65
query66 1712 476 356 356
query67 30047 29249 29723 29249
query68
query69 467 346 308 308
query70 1054 953 977 953
query71 308 280 271 271
query72 3005 2626 2426 2426
query73 828 771 427 427
query74 5062 4871 4741 4741
query75 2665 2601 2304 2304
query76 2297 1126 764 764
query77 404 425 353 353
query78 12115 12085 11558 11558
query79 1658 1067 733 733
query80 700 563 486 486
query81 467 280 244 244
query82 1368 162 126 126
query83 365 290 263 263
query84 266 146 123 123
query85 978 612 501 501
query86 423 340 324 324
query87 3445 3385 3212 3212
query88 3503 2658 2648 2648
query89 447 394 340 340
query90 1885 181 183 181
query91 179 169 137 137
query92 78 80 76 76
query93 1543 1400 843 843
query94 554 365 321 321
query95 676 401 446 401
query96 1040 812 356 356
query97 2729 2714 2527 2527
query98 236 231 226 226
query99 1127 1104 974 974
Total cold run time: 253426 ms
Total hot run time: 168565 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468516468
# FE UT Coverage Report
Increment line coverage ` 74.15% (109/147)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_1fe5df123ff286dc34a2f9ca8fef57c187bb95c4/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_1fe5df123ff286dc34a2f9ca8fef57c187bb95c4/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468495227
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468817697
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469389601
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469519362
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3254114916
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java:
##########
@@ -98,21 +102,47 @@ public void collect(Expression expression) {
expression.accept(this, new CollectorContext(statementContext, bottomPredicate));
}
+ public void collectWholeVariantExpression(Expression expression) {
+ CollectorContext context = new CollectorContext(statementContext, bottomPredicate);
+ context.setCollectVariantRoot(true);
+ expression.accept(this, context);
+ }
+
private Void continueCollectAccessPath(Expression expr, CollectorContext context) {
return expr.accept(this, context);
}
+ private void recordVariantRootAccessPath(SlotReference slotReference, CollectorContext context) {
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId,
+ new CollectAccessPathResult(
+ ImmutableList.of(slotReference.getName()),
+ context.bottomFilter, ColumnAccessPathType.DATA));
+ }
+
@Override
public Void visit(Expression expr, CollectorContext context) {
for (Expression child : expr.children()) {
- child.accept(this, new CollectorContext(context.statementContext, context.bottomFilter));
+ if (child.getDataType().isVariantType()
+ && (context.collectVariantRoot
Review Comment:
The generic visitor still does not force a root read for comparison predicates that consume a whole VARIANT but return BOOLEAN. For example, `WHERE v = cast('{"k":1,"other":2}' AS VARIANT) AND v['k'] IS NOT NULL` visits the `EqualTo` with an empty context; because the predicate result is BOOLEAN, this branch sends `v` through a fresh empty context and records no root `[v]` demand. The sibling nested predicate can then leave only `[v, k]` at the scan, so the equality is evaluated against a truncated VARIANT and can reject a matching row or accept a row that only matches the selected subpath. Please make whole-VARIANT-consuming comparison predicates force a root access and add coverage combining VARIANT equality/comparison with a sibling nested VARIANT predicate. This is distinct from the existing `variant_type` thread because comparison predicates are not handled by the new `GetVariantType` override and still fall through this default visitor.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469645889
<details>
<summary>TPC-DS: <b>Total hot run time: 170044 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 05227f64565f223e5d985a01530bda0cc6ec263b, data reload: false
query5 4314 662 499 499
query6 347 214 196 196
query7 4345 556 291 291
query8 324 233 248 233
query9 8838 4040 4016 4016
query10 452 352 298 298
query11 5757 2357 2237 2237
query12 190 128 124 124
query13 1302 610 441 441
query14 6030 5348 5061 5061
query14_1 4346 4345 4301 4301
query15 212 205 182 182
query16 1004 462 444 444
query17 1170 743 607 607
query18 2486 500 356 356
query19 224 207 172 172
query20 140 135 130 130
query21 213 136 123 123
query22 13561 13527 13319 13319
query23 17219 16391 15987 15987
query23_1 16188 16206 16129 16129
query24 7549 1768 1326 1326
query24_1 1312 1334 1292 1292
query25 584 502 428 428
query26 1303 324 179 179
query27 2693 580 343 343
query28 4460 1947 1967 1947
query29 991 621 511 511
query30 305 246 200 200
query31 1128 1069 955 955
query32 86 86 77 77
query33 553 371 306 306
query34 1215 1187 636 636
query35 762 787 730 730
query36 1366 1340 1209 1209
query37 153 105 92 92
query38 3204 3127 3032 3032
query39 929 923 894 894
query39_1 893 870 880 870
query40 221 143 129 129
query41 65 61 60 60
query42 106 109 108 108
query43 320 336 295 295
query44
query45 208 204 191 191
query46 1058 1193 712 712
query47 2339 2274 2274 2274
query48 399 411 292 292
query49 635 489 403 403
query50 1033 352 257 257
query51 4333 4428 4260 4260
query52 104 106 95 95
query53 251 301 215 215
query54 307 267 279 267
query55 93 92 84 84
query56 292 300 305 300
query57 1418 1395 1289 1289
query58 289 264 264 264
query59 1581 1722 1481 1481
query60 321 326 323 323
query61 156 182 154 154
query62 668 629 566 566
query63 244 204 212 204
query64 2386 791 631 631
query65
query66 1711 500 377 377
query67 29971 29892 29878 29878
query68
query69 462 336 314 314
query70 1048 995 997 995
query71 303 272 273 272
query72 2965 2700 2456 2456
query73 816 764 415 415
query74 5066 4934 4689 4689
query75 2633 2588 2249 2249
query76 2276 1128 752 752
query77 400 408 333 333
query78 12111 12175 11661 11661
query79 1474 1005 728 728
query80 1205 533 466 466
query81 507 283 237 237
query82 1388 157 121 121
query83 369 273 260 260
query84 272 142 113 113
query85 918 543 440 440
query86 432 324 329 324
query87 3484 3354 3169 3169
query88 3580 2658 2654 2654
query89 446 384 337 337
query90 1861 186 212 186
query91 179 165 141 141
query92 80 79 75 75
query93 1541 1471 887 887
query94 631 349 319 319
query95 657 400 350 350
query96 1036 757 325 325
query97 2737 2710 2557 2557
query98 236 228 229 228
query99 1103 1102 998 998
Total cold run time: 253879 ms
Total hot run time: 170044 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470140413
<details>
<summary>TPC-H: <b>Total hot run time: 31048 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit b7402399a565dc0a3e621ecb4d61cfd48a4eb2fc, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17767 3923 3848 3848
q2 q3 10827 1353 820 820
q4 4683 471 347 347
q5 7578 2184 2110 2110
q6 229 177 135 135
q7 910 757 631 631
q8 9354 1766 1611 1611
q9 6120 4875 4867 4867
q10 6444 2102 1808 1808
q11 438 274 246 246
q12 695 413 293 293
q13 18225 3337 2777 2777
q14 260 255 238 238
q15 q16 821 798 715 715
q17 989 973 998 973
q18 6794 5710 5591 5591
q19 1211 1290 1101 1101
q20 508 406 272 272
q21 5809 2602 2358 2358
q22 431 352 307 307
Total cold run time: 100093 ms
Total hot run time: 31048 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4317 4074 4109 4074
q2 q3 4488 4913 4304 4304
q4 2083 2200 1378 1378
q5 4372 4252 4256 4252
q6 226 178 129 129
q7 2032 1904 1703 1703
q8 2450 2102 2013 2013
q9 7850 7781 7769 7769
q10 4523 4463 4120 4120
q11 563 419 377 377
q12 706 849 591 591
q13 3299 3636 3009 3009
q14 304 320 280 280
q15 q16 734 732 640 640
q17 1350 1281 1310 1281
q18 7934 7264 7040 7040
q19 1116 1084 1101 1084
q20 2209 2202 1922 1922
q21 5303 4672 4542 4542
q22 515 470 411 411
Total cold run time: 56374 ms
Total hot run time: 50919 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470229489
# BE UT Coverage Report
Increment line coverage `75.50% (2668/3534)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/3c3488324f644f6c58190b59615a6cb681463643_3c3488324f644f6c58190b59615a6cb681463643/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/3c3488324f644f6c58190b59615a6cb681463643_3c3488324f644f6c58190b59615a6cb681463643/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.68% (20829/38804) |
| Line Coverage | 37.40% (197595/528382) |
| Region Coverage | 33.77% (154752/458224) |
| Branch Coverage | 34.88% (67857/194543) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4442703304
# BE UT Coverage Report
Increment line coverage `58.54% (1145/1956)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/081e39b4f33fcf00adabab60ee9644e710454994_081e39b4f33fcf00adabab60ee9644e710454994/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/081e39b4f33fcf00adabab60ee9644e710454994_081e39b4f33fcf00adabab60ee9644e710454994/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.62% (20696/38595) |
| Line Coverage | 37.28% (196060/525872) |
| Region Coverage | 33.71% (153472/455245) |
| Branch Coverage | 34.73% (67099/193221) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4442916133
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4444908246
# FE Regression Coverage Report
Increment line coverage ` 0.00% (0/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_cf7298fc89feb2f1764cbd30fbbe948468650fbc_merge_fe/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_cf7298fc89feb2f1764cbd30fbbe948468650fbc_merge_fe/report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4438312426
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4439528755
# BE UT Coverage Report
Increment line coverage `26.02% (458/1760)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/448355b9ec5afaa1afb2b0910fd3a6825e1007cf_448355b9ec5afaa1afb2b0910fd3a6825e1007cf/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/448355b9ec5afaa1afb2b0910fd3a6825e1007cf_448355b9ec5afaa1afb2b0910fd3a6825e1007cf/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.50% (20643/38582) |
| Line Coverage | 37.16% (195341/525649) |
| Region Coverage | 33.62% (152979/455047) |
| Branch Coverage | 34.58% (66774/193086) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440048292
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4440316021
# FE UT Coverage Report
Increment line coverage ` 25.00% (1/4)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/63192_1beb1746c3e4fbef9cb87898dc837d92107b692a/fe_increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/63192_1beb1746c3e4fbef9cb87898dc837d92107b692a/fe_report/index.html)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3234320583
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1784,140 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _column_ids = column_ids;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ DataTypePtr variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ variant_struct_type = make_nullable(variant_struct_type);
+ }
+ _variant_struct_field->data_type = variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::read_column_data(
+ ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
+ (void)root_node;
+ if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_VARIANT) {
+ return Status::Corruption(
+ "Wrong data type for column '{}', expected Variant type, actual type: {}.",
+ _field_schema->name, type->get_name());
+ }
+
+ const auto& variant_struct_type = _variant_struct_field->data_type;
+ ColumnPtr struct_column = variant_struct_type->create_column();
+ const size_t old_struct_rows = struct_column->size();
+ auto const_node = TableSchemaChangeHelper::ConstNode::get_instance();
+ RETURN_IF_ERROR(_struct_reader->read_column_data(struct_column, variant_struct_type, const_node,
+ filter_map, batch_size, read_rows, eof,
+ is_dict_filter, real_column_size));
+
+ const size_t new_struct_rows = struct_column->size() - old_struct_rows;
+ if (new_struct_rows == 0) {
+ return Status::OK();
+ }
+
+ MutableColumnPtr variant_column_ptr;
+ NullMap* null_map_ptr = nullptr;
+ auto mutable_column = doris_column->assume_mutable();
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = assert_cast<ColumnNullable*>(mutable_column.get());
+ variant_column_ptr = nullable_column->get_nested_column_ptr();
+ null_map_ptr = &nullable_column->get_null_map_data();
+ } else {
+ if (_field_schema->data_type->is_nullable()) {
+ return Status::Corruption("Not nullable column has null values in parquet file");
+ }
+ variant_column_ptr = std::move(mutable_column);
+ }
+ auto* variant_column = assert_cast<ColumnVariant*>(variant_column_ptr.get());
+
+ const IColumn* variant_struct_source = struct_column.get();
+ const NullMap* struct_null_map = nullptr;
+ if (const auto* nullable_struct = check_and_get_column<ColumnNullable>(variant_struct_source)) {
+ struct_null_map = &nullable_struct->get_null_map_data();
+ variant_struct_source = &nullable_struct->get_nested_column();
+ }
+ const auto& variant_struct_column = assert_cast<const ColumnStruct&>(*variant_struct_source);
+
+ const int value_idx = find_child_idx(*_field_schema, "value");
+ const int typed_value_idx = find_child_idx(*_field_schema, "typed_value");
+ if (value_idx < 0 && typed_value_idx >= 0 &&
+ can_direct_read_typed_value(_field_schema->children[typed_value_idx], false, _column_ids)) {
+ MutableColumnPtr batch_variant_column =
Review Comment:
Fixed in current head `7aef30cdbcdebd9735846948a34f710fcb054f2b`. The typed-only fast path now requires at least one selected direct typed leaf under `typed_value`; if pruning only selected root/metadata for a missing key, it falls back to the row-wise path and appends the correct number of missing/null VARIANT rows instead of copying past the default batch row. Added `ParquetVariantReaderTest.DirectTypedOnlyRequiresSelectedTypedLeaf` plus `typed_only_missing_field` regression and profile assertion that `v['missing']` reads only `v.metadata` and no `v.typed_value*`/`v.value`. Verified locally with `./run-be-ut.sh --run --filter="ParquetVariantReaderTest.*:NestedColumnAccessHelperTest.*"`, `./build.sh --be`, `./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d external_table_p0/tvf -s test_local_tvf_iceberg_variant -forceGenOut`, the same regression without `-forceGenOut`, `build-support/check-format.sh`, and `git diff --check`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4441989065
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3229990011
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +1125,382 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ _variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ _variant_struct_type = make_nullable(_variant_struct_type);
+ }
+ _variant_struct_field->data_type = _variant_struct_type;
+
+ RETURN_IF_ERROR(ParquetColumnReader::create(file, _variant_struct_field.get(), row_group,
+ _row_ranges, _ctz, _io_ctx, _struct_reader,
+ max_buf_size, col_offsets, state, in_collection,
+ column_ids, filter_column_ids));
+ _struct_reader->set_column_in_nested();
+ return Status::OK();
+}
+
+Status VariantColumnReader::_get_binary_field(const Field& field, std::string* value,
+ bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+Status VariantColumnReader::_field_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) const {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ json->append(field.to_debug_string(type->get_scale()));
+ return Status::OK();
+ case TYPE_FLOAT: {
+ const auto value = field.get<TYPE_FLOAT>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_DOUBLE: {
+ const auto value = field.get<TYPE_DOUBLE>();
+ json->append(std::isfinite(value) ? field.to_debug_string(type->get_scale()) : "null");
+ return Status::OK();
+ }
+ case TYPE_TIMEV2:
+ json->append(std::to_string(field.get<TYPE_TIMEV2>()));
+ return Status::OK();
+ case TYPE_DATE:
+ case TYPE_DATETIME:
Review Comment:
Shredded temporal typed values do not decode the same way as the unshredded VARIANT `value` path. Here DATE/DATETIME/TIMESTAMP fields are converted through `Field::to_debug_string()` and quoted as JSON strings (while TIMEV2 is emitted as a numeric value), but `decode_primitive()` emits Parquet VARIANT date/timestamp/time primitives as their raw numeric payloads. A valid file can store the same logical field in `value` for some rows and in `typed_value` for others, so Doris will materialize different `ColumnVariant` types depending only on shredding layout. Please normalize temporal primitives consistently across both paths and add a regression that reads the same DATE/TIME/TIMESTAMP value from unshredded and shredded VARIANT columns.
##########
be/src/format/parquet/parquet_variant_reader.cpp:
##########
@@ -0,0 +1,464 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_variant_reader.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <iomanip>
+#include <limits>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+namespace doris::parquet {
+
+namespace {
+
+struct VariantMetadata {
+ std::vector<std::string> dictionary;
+};
+
+uint64_t read_unsigned_le(const uint8_t* ptr, int size) {
+ uint64_t value = 0;
+ for (int i = 0; i < size; ++i) {
+ value |= static_cast<uint64_t>(ptr[i]) << (i * 8);
+ }
+ return value;
+}
+
+int64_t read_signed_le(const uint8_t* ptr, int size) {
+ uint64_t value = read_unsigned_le(ptr, size);
+ if (size < 8) {
+ uint64_t sign_bit = uint64_t {1} << (size * 8 - 1);
+ if ((value & sign_bit) != 0) {
+ uint64_t mask = ~((uint64_t {1} << (size * 8)) - 1);
+ value |= mask;
+ }
+ }
+ return static_cast<int64_t>(value);
+}
+
+Status require_available(const uint8_t* ptr, const uint8_t* end, size_t size,
+ std::string_view context) {
+ if (ptr + size > end) {
+ return Status::Corruption("Invalid Parquet VARIANT {} encoding", context);
+ }
+ return Status::OK();
+}
+
+void append_json_string(std::string_view value, std::string* json) {
+ json->push_back('"');
+ static constexpr char hex[] = "0123456789abcdef";
+ for (unsigned char c : value) {
+ switch (c) {
+ case '"':
+ json->append("\\\"");
+ break;
+ case '\\':
+ json->append("\\\\");
+ break;
+ case '\b':
+ json->append("\\b");
+ break;
+ case '\f':
+ json->append("\\f");
+ break;
+ case '\n':
+ json->append("\\n");
+ break;
+ case '\r':
+ json->append("\\r");
+ break;
+ case '\t':
+ json->append("\\t");
+ break;
+ default:
+ if (c < 0x20) {
+ json->append("\\u00");
+ json->push_back(hex[c >> 4]);
+ json->push_back(hex[c & 0x0f]);
+ } else {
+ json->push_back(static_cast<char>(c));
+ }
+ break;
+ }
+ }
+ json->push_back('"');
+}
+
+template <typename T>
+void append_floating_json(T value, std::string* json) {
+ if (!std::isfinite(value)) {
+ json->append("null");
+ return;
+ }
+ std::ostringstream oss;
+ oss << std::setprecision(std::numeric_limits<T>::max_digits10) << value;
+ json->append(oss.str());
+}
+
+std::string int128_to_string(__int128 value) {
+ if (value == 0) {
+ return "0";
+ }
+ bool negative = value < 0;
+ unsigned __int128 unsigned_value = negative ? static_cast<unsigned __int128>(-(value + 1)) + 1
+ : static_cast<unsigned __int128>(value);
+ std::string digits;
+ while (unsigned_value > 0) {
+ digits.push_back(static_cast<char>('0' + unsigned_value % 10));
+ unsigned_value /= 10;
+ }
+ if (negative) {
+ digits.push_back('-');
+ }
+ std::reverse(digits.begin(), digits.end());
+ return digits;
+}
+
+void append_decimal_json(__int128 unscaled, int scale, std::string* json) {
+ std::string value = int128_to_string(unscaled);
+ bool negative = !value.empty() && value[0] == '-';
+ std::string digits = negative ? value.substr(1) : value;
+ if (scale == 0) {
+ json->append(value);
+ return;
+ }
+ if (scale > 0) {
+ if (digits.size() <= static_cast<size_t>(scale)) {
+ digits.insert(0, static_cast<size_t>(scale) + 1 - digits.size(), '0');
+ }
+ digits.insert(digits.end() - scale, '.');
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ return;
+ }
+ if (negative) {
+ json->push_back('-');
+ }
+ json->append(digits);
+ json->append(static_cast<size_t>(-scale), '0');
+}
+
+void append_uuid_json(const uint8_t* ptr, std::string* json) {
+ static constexpr char hex[] = "0123456789abcdef";
+ json->push_back('"');
+ for (int i = 0; i < 16; ++i) {
+ if (i == 4 || i == 6 || i == 8 || i == 10) {
+ json->push_back('-');
+ }
+ json->push_back(hex[ptr[i] >> 4]);
+ json->push_back(hex[ptr[i] & 0x0f]);
+ }
+ json->push_back('"');
+}
+
+Status decode_metadata(const StringRef& metadata, VariantMetadata* result) {
+ const auto* ptr = reinterpret_cast<const uint8_t*>(metadata.data);
+ const auto* end = ptr + metadata.size;
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "metadata"));
+ uint8_t header = *ptr++;
+ uint8_t version = header & 0x0f;
+ if (version != 1) {
+ return Status::Corruption("Unsupported Parquet VARIANT metadata version {}", version);
+ }
+ int offset_size = ((header >> 6) & 0x03) + 1;
+ RETURN_IF_ERROR(require_available(ptr, end, offset_size, "metadata dictionary size"));
+ uint64_t dictionary_size = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+
+ size_t offsets_bytes = (dictionary_size + 1) * offset_size;
+ RETURN_IF_ERROR(require_available(ptr, end, offsets_bytes, "metadata dictionary offsets"));
+ std::vector<uint64_t> offsets(dictionary_size + 1);
+ for (uint64_t i = 0; i <= dictionary_size; ++i) {
+ offsets[i] = read_unsigned_le(ptr, offset_size);
+ ptr += offset_size;
+ if (i > 0 && offsets[i] < offsets[i - 1]) {
+ return Status::Corruption("Invalid Parquet VARIANT metadata dictionary offsets");
+ }
+ }
+
+ RETURN_IF_ERROR(require_available(ptr, end, offsets.back(), "metadata dictionary bytes"));
+ result->dictionary.clear();
+ result->dictionary.reserve(dictionary_size);
+ for (uint64_t i = 0; i < dictionary_size; ++i) {
+ result->dictionary.emplace_back(reinterpret_cast<const char*>(ptr + offsets[i]),
+ offsets[i + 1] - offsets[i]);
+ }
+ return Status::OK();
+}
+
+Status decode_value(const uint8_t* ptr, const uint8_t* end, const VariantMetadata& metadata,
+ std::string* json, const uint8_t** next);
+
+Status decode_primitive(uint8_t primitive_header, const uint8_t* ptr, const uint8_t* end,
+ std::string* json, const uint8_t** next) {
+ switch (primitive_header) {
+ case 0:
+ json->append("null");
+ *next = ptr;
+ return Status::OK();
+ case 1:
+ json->append("true");
+ *next = ptr;
+ return Status::OK();
+ case 2:
+ json->append("false");
+ *next = ptr;
+ return Status::OK();
+ case 3:
+ RETURN_IF_ERROR(require_available(ptr, end, 1, "int8 value"));
+ json->append(std::to_string(static_cast<int8_t>(*ptr)));
+ *next = ptr + 1;
+ return Status::OK();
+ case 4:
+ RETURN_IF_ERROR(require_available(ptr, end, 2, "int16 value"));
+ json->append(std::to_string(read_signed_le(ptr, 2)));
+ *next = ptr + 2;
+ return Status::OK();
+ case 5:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "int32 value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 6:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "int64 value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 7: {
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "double value"));
+ uint64_t bits = read_unsigned_le(ptr, 8);
+ double value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 8;
+ return Status::OK();
+ }
+ case 8:
+ case 9:
+ case 10: {
+ int value_size = primitive_header == 8 ? 4 : primitive_header == 9 ? 8 : 16;
+ RETURN_IF_ERROR(require_available(ptr, end, 1 + value_size, "decimal value"));
+ int scale = static_cast<int8_t>(*ptr++);
+ __int128 unscaled = 0;
+ if (value_size == 16) {
+ unsigned __int128 unsigned_value = 0;
+ for (int i = 15; i >= 0; --i) {
+ unsigned_value <<= 8;
+ unsigned_value |= ptr[i];
+ }
+ if ((ptr[15] & 0x80) != 0) {
+ unscaled = -static_cast<__int128>((~unsigned_value) + 1);
+ } else {
+ unscaled = static_cast<__int128>(unsigned_value);
+ }
+ } else {
+ unscaled = read_signed_le(ptr, value_size);
+ }
+ append_decimal_json(unscaled, scale, json);
+ *next = ptr + value_size;
+ return Status::OK();
+ }
+ case 11:
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "date value"));
+ json->append(std::to_string(read_signed_le(ptr, 4)));
+ *next = ptr + 4;
+ return Status::OK();
+ case 12:
+ case 13:
+ case 17:
+ case 18:
+ case 19:
+ RETURN_IF_ERROR(require_available(ptr, end, 8, "time or timestamp value"));
+ json->append(std::to_string(read_signed_le(ptr, 8)));
+ *next = ptr + 8;
+ return Status::OK();
+ case 14: {
+ RETURN_IF_ERROR(require_available(ptr, end, 4, "float value"));
+ uint32_t bits = static_cast<uint32_t>(read_unsigned_le(ptr, 4));
+ float value;
+ std::memcpy(&value, &bits, sizeof(value));
+ append_floating_json(value, json);
+ *next = ptr + 4;
+ return Status::OK();
+ }
+ case 15:
+ case 16: {
Review Comment:
Parquet VARIANT `binary` is not equivalent to `string`, and the payload is not required to be valid UTF-8. This branch serializes primitive 15 by passing the raw bytes through JSON string escaping, which only escapes control characters; bytes such as `0xff` produce invalid JSON/UTF-8 (or are later treated as an ordinary Doris string via the invalid-JSON fallback). The shredded path has the same semantic gap for `TYPE_VARBINARY` in `_field_to_json()`. Please define a stable representation or reject unsupported binary values explicitly, and cover a non-UTF8 binary VARIANT value so valid Parquet files do not silently change type or fail depending on parser settings.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4436370451
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434079915
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4435137424
<details>
<summary>TPC-DS: <b>Total hot run time: 171345 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 1ef3cb842d3c89759708e6778d83462779d139bb, data reload: false
query5 4340 661 538 538
query6 357 221 212 212
query7 4212 561 312 312
query8 330 227 225 225
query9 8841 4065 4065 4065
query10 491 350 303 303
query11 6160 2438 2226 2226
query12 204 130 133 130
query13 1269 581 450 450
query14 7916 5393 5104 5104
query14_1 4401 4395 4400 4395
query15 258 206 186 186
query16 1033 452 446 446
query17 1359 763 628 628
query18 2783 474 351 351
query19 317 204 166 166
query20 149 132 133 132
query21 218 134 114 114
query22 13582 13617 13431 13431
query23 17172 16244 16642 16244
query23_1 16246 16278 16316 16278
query24 7316 1799 1386 1386
query24_1 1377 1410 1435 1410
query25 625 526 467 467
query26 826 321 179 179
query27 2895 594 345 345
query28 4442 1963 1948 1948
query29 981 649 547 547
query30 300 233 198 198
query31 1090 1073 941 941
query32 86 73 70 70
query33 547 356 301 301
query34 1133 1115 629 629
query35 772 798 672 672
query36 1353 1295 1132 1132
query37 149 101 89 89
query38 3200 3132 3065 3065
query39 919 920 895 895
query39_1 892 895 898 895
query40 227 167 144 144
query41 70 68 66 66
query42 111 111 114 111
query43 336 337 294 294
query44
query45 214 205 194 194
query46 1062 1195 749 749
query47 2259 2277 2096 2096
query48 417 433 320 320
query49 630 548 439 439
query50 699 291 226 226
query51 4351 4272 4175 4175
query52 107 108 94 94
query53 259 282 207 207
query54 325 286 270 270
query55 95 91 86 86
query56 324 331 325 325
query57 1385 1375 1309 1309
query58 288 272 273 272
query59 1595 1641 1411 1411
query60 352 338 326 326
query61 156 169 156 156
query62 661 614 563 563
query63 241 205 203 203
query64 2097 792 667 667
query65
query66 1621 499 389 389
query67 30135 30000 29902 29902
query68
query69 470 337 304 304
query70 1044 995 938 938
query71 326 279 278 278
query72 3202 2748 2442 2442
query73 855 729 444 444
query74 5077 4949 4716 4716
query75 2784 2650 2330 2330
query76 2287 1121 740 740
query77 414 426 356 356
query78 13051 12873 12367 12367
query79 1531 955 761 761
query80 1382 564 521 521
query81 521 285 241 241
query82 1118 157 127 127
query83 351 281 253 253
query84 263 139 112 112
query85 897 508 460 460
query86 439 320 313 313
query87 3439 3399 3198 3198
query88 3560 2688 2674 2674
query89 438 380 340 340
query90 1903 182 187 182
query91 178 171 140 140
query92 82 75 72 72
query93 1057 947 575 575
query94 718 362 298 298
query95 664 379 437 379
query96 970 745 337 337
query97 2687 2690 2580 2580
query98 237 247 229 229
query99 1125 1089 971 971
Total cold run time: 256475 ms
Total hot run time: 171345 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4435509008
# BE UT Coverage Report
Increment line coverage `6.41% (86/1342)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/1ef3cb842d3c89759708e6778d83462779d139bb_1ef3cb842d3c89759708e6778d83462779d139bb/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/1ef3cb842d3c89759708e6778d83462779d139bb_1ef3cb842d3c89759708e6778d83462779d139bb/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.48% (20638/38591) |
| Line Coverage | 37.13% (195086/525473) |
| Region Coverage | 33.56% (152619/454757) |
| Branch Coverage | 34.48% (66513/192876) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437506229
<details>
<summary>TPC-DS: <b>Total hot run time: 170907 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 747e7defc5a69683b7685801020ca02080d75269, data reload: false
query5 4323 676 523 523
query6 347 234 203 203
query7 4229 598 300 300
query8 330 233 219 219
query9 8814 4029 4055 4029
query10 455 341 306 306
query11 5796 2390 2208 2208
query12 191 139 125 125
query13 1380 578 411 411
query14 5961 5336 5025 5025
query14_1 4344 4345 4319 4319
query15 210 208 179 179
query16 993 421 428 421
query17 1132 763 613 613
query18 2441 478 362 362
query19 212 193 159 159
query20 140 129 127 127
query21 211 137 116 116
query22 13614 13822 14694 13822
query23 17494 16545 16224 16224
query23_1 16328 16322 16306 16306
query24 7440 1747 1365 1365
query24_1 1377 1333 1340 1333
query25 608 524 444 444
query26 1309 310 171 171
query27 2741 609 357 357
query28 4421 1970 1931 1931
query29 1035 656 532 532
query30 308 228 199 199
query31 1104 1078 951 951
query32 91 74 71 71
query33 573 363 295 295
query34 1193 1120 637 637
query35 778 767 672 672
query36 1333 1335 1160 1160
query37 155 110 90 90
query38 3228 3119 3062 3062
query39 931 928 886 886
query39_1 894 869 870 869
query40 240 165 141 141
query41 121 75 71 71
query42 107 104 109 104
query43 321 318 279 279
query44
query45 210 197 187 187
query46 1038 1223 698 698
query47 2240 2275 2184 2184
query48 414 402 296 296
query49 636 518 446 446
query50 724 282 217 217
query51 4314 4258 4244 4244
query52 105 108 93 93
query53 254 271 199 199
query54 301 276 258 258
query55 94 90 81 81
query56 309 300 303 300
query57 1411 1374 1293 1293
query58 296 265 279 265
query59 1523 1574 1404 1404
query60 342 338 321 321
query61 162 159 151 151
query62 656 611 553 553
query63 245 199 201 199
query64 2438 841 713 713
query65
query66 1761 506 387 387
query67 29981 29401 29903 29401
query68
query69 478 344 305 305
query70 1027 986 1009 986
query71 303 290 268 268
query72 2991 2739 2452 2452
query73 853 763 419 419
query74 5070 4929 4733 4733
query75 2756 2668 2330 2330
query76 2319 1134 762 762
query77 435 439 357 357
query78 12954 12941 12251 12251
query79 1510 963 734 734
query80 1358 572 481 481
query81 520 283 236 236
query82 992 158 124 124
query83 349 289 248 248
query84 266 138 107 107
query85 905 516 448 448
query86 446 309 345 309
query87 3401 3359 3228 3228
query88 3597 2684 2676 2676
query89 438 380 337 337
query90 1952 192 187 187
query91 178 173 140 140
query92 82 76 74 74
query93 1182 977 550 550
query94 728 349 298 298
query95 653 468 344 344
query96 1075 752 319 319
query97 2697 2696 2592 2592
query98 245 244 245 244
query99 1095 1113 982 982
Total cold run time: 254489 ms
Total hot run time: 170907 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437648924
<details>
<summary>TPC-H: <b>Total hot run time: 29702 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit cb90a9e5d9e7319be77f506d54888d62c5dc36b7, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17627 3974 3900 3900
q2 q3 10723 882 614 614
q4 4666 460 345 345
q5 7458 1328 1147 1147
q6 191 168 138 138
q7 912 954 752 752
q8 9295 1414 1286 1286
q9 5650 5380 5326 5326
q10 6252 2081 1825 1825
q11 465 277 261 261
q12 628 407 302 302
q13 18151 3336 2742 2742
q14 293 286 267 267
q15 q16 874 875 794 794
q17 962 988 813 813
q18 6482 5679 5545 5545
q19 1200 1257 1075 1075
q20 484 402 263 263
q21 4411 2367 1969 1969
q22 471 402 338 338
Total cold run time: 97195 ms
Total hot run time: 29702 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4675 4764 5064 4764
q2 q3 4682 4790 4210 4210
q4 2109 2157 1409 1409
q5 5031 5086 5215 5086
q6 194 173 133 133
q7 2075 1803 1628 1628
q8 3346 3101 3151 3101
q9 8394 8361 8440 8361
q10 4499 4532 4255 4255
q11 601 424 389 389
q12 703 756 521 521
q13 3280 3654 2943 2943
q14 302 302 378 302
q15 q16 794 795 694 694
q17 1341 1313 1271 1271
q18 8102 7162 7164 7162
q19 1164 1170 1143 1143
q20 2288 2290 1945 1945
q21 6241 5516 4945 4945
q22 557 517 426 426
Total cold run time: 60378 ms
Total hot run time: 54688 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437619151
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4437619739
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469626270
<details>
<summary>TPC-H: <b>Total hot run time: 30841 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 05227f64565f223e5d985a01530bda0cc6ec263b, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17742 3907 3853 3853
q2 q3 10761 1424 788 788
q4 4686 468 360 360
q5 7576 2296 2164 2164
q6 306 178 136 136
q7 977 781 626 626
q8 9348 1791 1595 1595
q9 6882 4915 4952 4915
q10 6459 2121 1784 1784
q11 437 273 245 245
q12 690 418 289 289
q13 18202 3381 2785 2785
q14 266 253 234 234
q15 q16 822 786 702 702
q17 977 948 990 948
q18 6994 5828 5514 5514
q19 1238 1217 1025 1025
q20 487 408 259 259
q21 5737 2648 2311 2311
q22 424 365 308 308
Total cold run time: 101011 ms
Total hot run time: 30841 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4266 4212 4230 4212
q2 q3 4517 4896 4295 4295
q4 2070 2223 1359 1359
q5 4429 4329 4340 4329
q6 306 333 150 150
q7 2025 1874 1604 1604
q8 2470 2264 2202 2202
q9 7874 7829 7718 7718
q10 4486 4468 4026 4026
q11 581 454 542 454
q12 722 725 510 510
q13 3362 3638 3033 3033
q14 305 310 282 282
q15 q16 732 727 673 673
q17 1382 1305 1342 1305
q18 8052 7404 7036 7036
q19 1107 1100 1091 1091
q20 2215 2205 1923 1923
q21 5308 4769 4532 4532
q22 518 472 432 432
Total cold run time: 56727 ms
Total hot run time: 51166 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4457027389
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3252707483
##########
be/src/format/table/hive/hive_parquet_nested_column_utils.cpp:
##########
@@ -18,20 +18,294 @@
#include "format/table/hive/hive_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
+#include "core/data_type/data_type_nullable.h"
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return typed_value != nullptr;
+ }
Review Comment:
The Hive/local pruning helper has the same residual-only gap. For a valid shredded field group like `typed_value.metric.value` with no `typed_value.metric.typed_value`, an access such as `v['metric']['x']` reaches `extract_variant_typed_nested_column_ids(metric, [['x']])`; this predicate returns false because `typed_value == nullptr`, the generic struct path finds no child `x`, and the caller falls back to top-level `v.value` instead of selecting `v.typed_value.metric.value`. That silently prunes the only column containing the selected field's residual object. Please treat `value`-only shredded field groups as variant-like and add a pruning/profile assertion for this case.
##########
be/src/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp:
##########
@@ -18,21 +18,295 @@
#include "format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <iostream>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
+#include "core/data_type/data_type_nullable.h"
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return typed_value != nullptr;
+ }
Review Comment:
The duplicated Iceberg pruning helper also misses value-only field-level residual groups. In Iceberg Parquet, a shredded field can have `typed_value.<field>.value` without a nested `typed_value`; deeper accesses under that field currently fall through to top-level `v.value`, so Iceberg scans can skip the only residual column for that field. Please apply the same value-only wrapper recognition and regression/profile coverage here as in the Hive/local helper.
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +119,857 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr;
Review Comment:
This rejects a valid field-level residual-only shredded VARIANT wrapper. A shredded object field can be represented as a group containing only `value` when there is no typed shard for that field, e.g. `v.typed_value.metric { optional binary value; }`. With `has_value && typed_value == nullptr`, `is_variant_wrapper_field()` returns false, so `shredded_field_to_variant_map()` treats the group as an ordinary struct, appends a user path named `value`, and then tries to materialize the raw encoded bytes instead of decoding them with the inherited metadata. The result is either an unsupported/corrupt value or the wrong path (`metric.value` instead of `metric`). Please recognize `value`-only field-level wrappers when inherited metadata is available and add coverage for a shredded field whose residual is stored only in `typed_value.<field>.value`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3252833924
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +123,1069 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper,
+ bool allow_value_only_wrapper = false) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr || allow_value_only_wrapper;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ JsonBinaryValue empty_object;
+ RETURN_IF_ERROR(empty_object.from_json_string("{}"));
+ (*values)[path] = FieldWithDataType {.field = Field::create_field<TYPE_JSONB>(JsonbField(
+ empty_object.value(), empty_object.size())),
+ .base_scalar_type_id = TYPE_JSONB};
+ return Status::OK();
+}
+
+static bool is_empty_object_marker(const FieldWithDataType& value) {
+ if (value.field.get_type() != TYPE_JSONB) {
+ return false;
+ }
+ const auto& jsonb = value.field.get<TYPE_JSONB>();
+ const JsonbDocument* document = nullptr;
+ Status st =
+ JsonbDocument::checkAndCreateDocument(jsonb.get_value(), jsonb.get_size(), &document);
+ if (!st.ok() || document == nullptr || document->getValue() == nullptr ||
+ !document->getValue()->isObject()) {
+ return false;
+ }
+ return document->getValue()->unpack<ObjectVal>()->numElem() == 0;
+}
+
+static Status collect_empty_object_markers(const rapidjson::Value& value, PathInDataBuilder* path,
+ VariantMap* values) {
+ if (!value.IsObject()) {
+ return Status::OK();
+ }
+ if (value.MemberCount() == 0) {
+ return insert_empty_object_marker(path->build(), values);
+ }
+ for (auto it = value.MemberBegin(); it != value.MemberEnd(); ++it) {
+ if (it->value.IsObject()) {
+ path->append(std::string_view(it->name.GetString(), it->name.GetStringLength()), false);
+ RETURN_IF_ERROR(collect_empty_object_markers(it->value, path, values));
+ path->pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+static Status add_empty_object_markers_from_json(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ if (json.find("{}") == std::string::npos) {
+ return Status::OK();
+ }
+ rapidjson::Document document;
+ document.Parse(json.data(), json.size());
+ if (document.HasParseError()) {
+ return Status::Corruption("Invalid Parquet VARIANT decoded JSON");
+ }
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ return collect_empty_object_markers(document, &path, values);
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (!parsed.is_null()) {
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ }
+ RETURN_IF_ERROR(add_empty_object_markers_from_json(json, prefix, values));
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static bool path_has_prefix(const PathInData& path, const PathInData& prefix) {
+ const auto& parts = path.get_parts();
+ const auto& prefix_parts = prefix.get_parts();
+ if (parts.size() < prefix_parts.size()) {
+ return false;
+ }
+ for (size_t i = 0; i < prefix_parts.size(); ++i) {
+ if (parts[i] != prefix_parts[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool has_descendant_path(const VariantMap& values, const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ return std::ranges::any_of(values, [&](const auto& entry) {
+ const auto& path = entry.first;
+ return path.get_parts().size() > prefix_size && path_has_prefix(path, prefix);
+ });
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* values,
+ const VariantMap& shadowing_values) {
+ for (auto it = values->begin(); it != values->end();) {
+ if (is_empty_object_marker(it->second) &&
+ (has_descendant_path(*values, it->first) ||
+ has_descendant_path(shadowing_values, it->first))) {
+ it = values->erase(it);
+ continue;
+ }
+ ++it;
+ }
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* value_values,
+ VariantMap* typed_values) {
+ erase_shadowed_empty_object_markers(value_values, *typed_values);
+ erase_shadowed_empty_object_markers(typed_values, *value_values);
+}
+
+static Status check_no_shredded_value_typed_duplicates(const VariantMap& value_values,
+ const VariantMap& typed_values,
+ const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ for (const auto& value_entry : value_values) {
+ const auto& value_path = value_entry.first;
+ if (!path_has_prefix(value_path, prefix)) {
+ continue;
+ }
+ if (value_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(value_entry.second) &&
+ !has_descendant_path(typed_values, value_path)) {
+ continue;
+ }
+ if (!typed_values.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT residual value conflicts with typed_value at path {}",
+ value_path.get_path());
+ }
+ continue;
+ }
+ for (const auto& typed_entry : typed_values) {
+ const auto& typed_path = typed_entry.first;
+ if (!path_has_prefix(typed_path, prefix)) {
+ continue;
+ }
+ if (typed_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(typed_entry.second) &&
+ !has_descendant_path(value_values, typed_path)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ if (value_path.get_parts()[prefix_size] == typed_path.get_parts()[prefix_size]) {
+ if (value_path == typed_path && is_empty_object_marker(value_entry.second) &&
+ is_empty_object_marker(typed_entry.second)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+static bool has_direct_typed_parent_null(const std::vector<const NullMap*>& null_maps, size_t row) {
+ return std::ranges::any_of(null_maps, [&](const NullMap* null_map) {
+ DCHECK_LT(row, null_map->size());
+ return (*null_map)[row];
+ });
+}
+
+static void insert_direct_typed_leaf_range(const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps,
+ IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ nullable_leaf.get_nested_column().insert_range_from(*value_column, start, rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static bool is_temporal_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static DataTypePtr direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return std::make_shared<DataTypeInt64>();
+ }
+ return type;
+}
+
+static bool contains_temporal_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_temporal_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static int64_t direct_temporal_variant_value(PrimitiveType type, const IColumn& column,
+ size_t row) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ return static_cast<int64_t>(
+ std::llround(assert_cast<const ColumnTimeV2&>(column).get_data()[row]));
+ case TYPE_DATE:
+ return variant_date_value(assert_cast<const ColumnDate&>(column).get_data()[row]);
+ case TYPE_DATETIME:
+ return variant_datetime_value(assert_cast<const ColumnDateTime&>(column).get_data()[row]);
+ case TYPE_DATEV2:
+ return variant_date_value(assert_cast<const ColumnDateV2&>(column).get_data()[row]);
+ case TYPE_DATETIMEV2:
+ return variant_datetime_value(assert_cast<const ColumnDateTimeV2&>(column).get_data()[row]);
+ case TYPE_TIMESTAMPTZ:
+ return variant_datetime_value(
+ assert_cast<const ColumnTimeStampTz&>(column).get_data()[row]);
+ default:
+ DORIS_CHECK(false);
+ return 0;
+ }
+}
+
+static void insert_direct_typed_temporal_leaf_range(
+ PrimitiveType type, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnInt64&>(nullable_leaf.get_nested_column()).get_data();
+ data.reserve(data.size() + rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ data.push_back(direct_temporal_variant_value(type, *value_column, row));
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ return true;
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return !contains_temporal_variant_leaf_type(array_type->get_nested_type()) &&
+ is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool has_selected_direct_typed_leaf(const FieldSchema& field_schema,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return false;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ return has_selected_direct_typed_leaf(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_direct_typed_leaf(child, true, column_ids);
+ });
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool can_use_direct_typed_only_value(const FieldSchema& variant_field,
+ const std::set<uint64_t>& column_ids) {
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ return (value_idx < 0 || !has_selected_column(variant_field.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ has_selected_direct_typed_leaf(variant_field.children[typed_value_idx], false,
+ column_ids) &&
+ can_direct_read_typed_value(variant_field.children[typed_value_idx], false, column_ids);
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ value->precision = type->get_precision();
+ value->scale = type->get_scale();
+ return Status::OK();
+ case TYPE_FLOAT: {
+ const auto float_value = field.get<TYPE_FLOAT>();
+ if (!std::isfinite(float_value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point typed_value is not supported");
+ }
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+ }
+ case TYPE_DOUBLE: {
+ const auto double_value = field.get<TYPE_DOUBLE>();
+ if (!std::isfinite(double_value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point typed_value is not supported");
+ }
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+ }
+ case TYPE_TIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ static_cast<int64_t>(std::llround(field.get<TYPE_TIMEV2>())));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATE:
+ value->field = Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATE>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIME:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIME>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATEV2:
+ value->field =
+ Field::create_field<TYPE_BIGINT>(variant_date_value(field.get<TYPE_DATEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_DATETIMEV2:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_DATETIMEV2>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_TIMESTAMPTZ:
+ value->field = Field::create_field<TYPE_BIGINT>(
+ variant_datetime_value(field.get<TYPE_TIMESTAMPTZ>()));
+ value->base_scalar_type_id = TYPE_BIGINT;
+ return Status::OK();
+ case TYPE_VARBINARY:
+ return Status::NotSupported("Parquet VARIANT binary typed_value is not supported");
+ default:
+ return Status::Corruption("Unsupported Parquet VARIANT typed_value Doris type {}",
+ type->get_name());
+ }
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present);
+
+static Status serialize_field_to_json(const DataTypePtr& data_type, const Field& field,
+ std::string* json) {
+ MutableColumnPtr column = data_type->create_column();
+ column->insert(field);
+
+ auto json_column = ColumnString::create();
+ VectorBufferWriter writer(*json_column);
+ auto serde = data_type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ RETURN_IF_ERROR(serde->serialize_one_cell_to_json(*column, 0, writer, options));
+ writer.commit();
+ *json = json_column->get_data_at(0).to_string();
+ return Status::OK();
+}
+
+static Status scalar_typed_value_to_json(const FieldSchema& field_schema, const Field& field,
+ std::string* json, bool* present) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(field_schema, field, &value, present));
+ if (!*present) {
+ return Status::OK();
+ }
+ if (value.field.is_null()) {
+ *json = "null";
+ return Status::OK();
+ }
+
+ DataTypePtr json_type;
+ if (value.base_scalar_type_id != PrimitiveType::INVALID_TYPE) {
+ json_type = DataTypeFactory::instance().create_data_type(value.base_scalar_type_id, false,
+ value.precision, value.scale);
+ } else {
+ json_type = remove_nullable(field_schema.data_type);
+ }
+ return serialize_field_to_json(json_type, value.field, json);
+}
+
+static Status variant_to_json(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, std::string* json,
+ bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ std::string typed_json;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_json(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, &typed_json,
+ &typed_present));
+ }
+
+ std::string value_json;
+ bool value_present = false;
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ }
+ }
+
+ if (value_present && typed_present) {
+ VariantMap value_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, PathInData(), &value_values));
+ VariantMap typed_values;
+ RETURN_IF_ERROR(parse_json_to_variant_map(typed_json, PathInData(), &typed_values));
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto root_value = value_values.find(PathInData());
+ if (root_value != value_values.end() && !is_empty_object_marker(root_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, PathInData()));
+ value_values.merge(std::move(typed_values));
+ RETURN_IF_ERROR(variant_map_to_json(std::move(value_values), json));
+ *present = true;
+ return Status::OK();
+ }
+
+ if (typed_present) {
+ *json = std::move(typed_json);
+ *present = true;
+ return Status::OK();
+ }
+ if (value_present) {
+ *json = std::move(value_json);
+ *present = true;
+ return Status::OK();
+ }
+
+ *present = false;
+ return Status::OK();
+}
+
+static Status shredded_field_to_json(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, std::string* json, bool* present,
+ bool allow_scalar_typed_value_only_wrapper) {
+ if (is_variant_wrapper_field(field_schema, allow_scalar_typed_value_only_wrapper, true)) {
+ return variant_to_json(field_schema, field, &metadata, json, present);
+ }
+ return typed_value_to_json(field_schema, field, metadata, json, present);
+}
+
+static Status typed_array_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ if (typed_value_field.children.empty()) {
+ return Status::Corruption("Parquet VARIANT array typed_value has no element schema");
+ }
+
+ const auto& elements = field.get<TYPE_ARRAY>();
+ const auto& element_schema = typed_value_field.children[0];
+ json->clear();
+ json->push_back('[');
+ for (size_t i = 0; i < elements.size(); ++i) {
+ if (i != 0) {
+ json->push_back(',');
+ }
+ std::string element_json;
+ bool element_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(element_schema, elements[i], metadata, &element_json,
+ &element_present, true));
+ if (element_present) {
+ json->append(element_json);
+ } else {
+ return Status::Corruption("Parquet VARIANT array typed_value element is missing");
+ }
+ }
+ json->push_back(']');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_struct_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+
+ const auto& fields = field.get<TYPE_STRUCT>();
+ json->clear();
+ json->push_back('{');
+ bool first = true;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ std::string child_json;
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_json(typed_value_field.children[i], fields[i], metadata,
+ &child_json, &child_present, false));
+ if (!child_present) {
+ continue;
+ }
+ if (!first) {
+ json->push_back(',');
+ }
+ append_json_string(typed_value_field.children[i].name, json);
+ json->push_back(':');
+ json->append(child_json);
+ first = false;
+ }
+ json->push_back('}');
+ *present = true;
+ return Status::OK();
+}
+
+static Status typed_value_to_json(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, std::string* json, bool* present) {
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ switch (typed_type->get_primitive_type()) {
+ case TYPE_STRUCT:
+ return typed_struct_to_json(typed_value_field, field, metadata, json, present);
+ case TYPE_ARRAY:
+ return typed_array_to_json(typed_value_field, field, metadata, json, present);
+ default:
+ return scalar_typed_value_to_json(typed_value_field, field, json, present);
+ }
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present);
+
+static Status variant_to_variant_map(const FieldSchema& variant_field, const Field& field,
+ const std::string* inherited_metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const auto& fields = field.get<TYPE_STRUCT>();
+ const int metadata_idx = find_child_idx(variant_field, "metadata");
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+
+ std::string metadata;
+ bool has_metadata = false;
+ if (inherited_metadata != nullptr) {
+ metadata = *inherited_metadata;
+ has_metadata = true;
+ }
+ if (metadata_idx >= 0) {
+ bool metadata_present = false;
+ RETURN_IF_ERROR(get_binary_field(fields[metadata_idx], &metadata, &metadata_present));
+ has_metadata = metadata_present;
+ }
+
+ VariantMap value_values;
+ bool value_present = false;
+ const PathInData current_path = path->build();
+ if (value_idx >= 0) {
+ std::string value;
+ RETURN_IF_ERROR(get_binary_field(fields[value_idx], &value, &value_present));
+ if (value_present) {
+ if (!has_metadata) {
+ return Status::Corruption("Parquet VARIANT value is present without metadata");
+ }
+ std::string value_json;
+ RETURN_IF_ERROR(parquet::decode_variant_to_json(
+ StringRef(metadata.data(), metadata.size()),
+ StringRef(value.data(), value.size()), &value_json));
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, current_path, &value_values));
+ }
+ }
+
+ VariantMap typed_values;
+ bool typed_present = false;
+ if (typed_value_idx >= 0) {
+ RETURN_IF_ERROR(typed_value_to_variant_map(variant_field.children[typed_value_idx],
+ fields[typed_value_idx], metadata, path,
+ &typed_values, &typed_present));
+ }
+
+ erase_shadowed_empty_object_markers(&value_values, &typed_values);
+ auto current_value = value_values.find(current_path);
+ if (value_present && typed_present && current_value != value_values.end() &&
+ !is_empty_object_marker(current_value->second)) {
+ return Status::Corruption(
+ "Parquet VARIANT has conflicting non-object value and typed_value");
+ }
+ RETURN_IF_ERROR(
+ check_no_shredded_value_typed_duplicates(value_values, typed_values, current_path));
+ values->merge(std::move(value_values));
+ values->merge(std::move(typed_values));
+ *present = value_present || typed_present;
+ return Status::OK();
+}
+
+static Status shredded_field_to_variant_map(const FieldSchema& field_schema, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (is_variant_wrapper_field(field_schema, false, true)) {
+ return variant_to_variant_map(field_schema, field, &metadata, path, values, present);
+ }
+ return typed_value_to_variant_map(field_schema, field, metadata, path, values, present);
+}
+
+static Status typed_value_to_variant_map(const FieldSchema& typed_value_field, const Field& field,
+ const std::string& metadata, PathInDataBuilder* path,
+ VariantMap* values, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ const DataTypePtr& typed_type = remove_nullable(typed_value_field.data_type);
+ if (typed_type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& fields = field.get<TYPE_STRUCT>();
+ *present = true;
+ bool has_present_child = false;
+ for (int i = 0; i < typed_value_field.children.size(); ++i) {
+ path->append(typed_value_field.children[i].name, false);
+ bool child_present = false;
+ RETURN_IF_ERROR(shredded_field_to_variant_map(typed_value_field.children[i], fields[i],
+ metadata, path, values, &child_present));
+ has_present_child |= child_present;
+ path->pop_back();
+ }
+ if (!has_present_child) {
+ RETURN_IF_ERROR(insert_empty_object_marker(path->build(), values));
+ }
+ return Status::OK();
+ }
+ if (typed_type->get_primitive_type() == TYPE_ARRAY) {
+ std::string value_json;
+ RETURN_IF_ERROR(
+ typed_value_to_json(typed_value_field, field, metadata, &value_json, present));
+ if (*present) {
+ RETURN_IF_ERROR(parse_json_to_variant_map(value_json, path->build(), values));
+ }
+ return Status::OK();
+ }
+
+ FieldWithDataType value;
+ RETURN_IF_ERROR(field_to_variant_field(typed_value_field, field, &value, present));
+ if (*present) {
+ (*values)[path->build()] = std::move(value);
+ }
+ return Status::OK();
+}
+
+static Status append_direct_typed_column_to_batch(const FieldSchema& field_schema,
+ const IColumn& column, size_t start, size_t rows,
+ PathInDataBuilder* path, ColumnVariant* batch,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids,
+ std::vector<const NullMap*> parent_null_maps) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return Status::OK();
+ }
+
+ const IColumn* value_column = &column;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ parent_null_maps.push_back(&nullable_column->get_null_map_data());
+ value_column = &nullable_column->get_nested_column();
+ }
+
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ const auto& typed_struct = assert_cast<const ColumnStruct&>(*value_column);
+ return append_direct_typed_column_to_batch(
+ field_schema.children[typed_value_idx], typed_struct.get_column(typed_value_idx),
+ start, rows, path, batch, false, column_ids, parent_null_maps);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ const auto& struct_column = assert_cast<const ColumnStruct&>(*value_column);
+ for (int i = 0; i < field_schema.children.size(); ++i) {
+ if (!has_selected_column(field_schema.children[i], column_ids)) {
+ continue;
+ }
+ path->append(field_schema.children[i].name, false);
+ RETURN_IF_ERROR(append_direct_typed_column_to_batch(
+ field_schema.children[i], struct_column.get_column(i), start, rows, path, batch,
+ true, column_ids, parent_null_maps));
+ path->pop_back();
+ }
+ return Status::OK();
+ }
+
+ DataTypePtr variant_leaf_type = make_nullable(direct_variant_leaf_type(field_schema.data_type));
+ MutableColumnPtr variant_leaf = variant_leaf_type->create_column();
+ variant_leaf->insert_default();
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ insert_direct_typed_temporal_leaf_range(type->get_primitive_type(), *value_column, start,
+ rows, parent_null_maps, variant_leaf.get());
+ } else {
+ insert_direct_typed_leaf_range(*value_column, start, rows, parent_null_maps,
+ variant_leaf.get());
+ }
+ if (!batch->add_sub_column(path->build(), std::move(variant_leaf), variant_leaf_type)) {
+ return Status::Corruption("Failed to add Parquet VARIANT typed subcolumn {}",
+ path->build().get_path());
+ }
+ return Status::OK();
+}
+
+#ifdef BE_TEST
+namespace parquet_variant_reader_test {
+bool can_direct_read_typed_value_for_test(const FieldSchema& typed_value_field) {
Review Comment:
This array branch reconstructs typed arrays by serializing the typed column to JSON and reparsing it into a `VariantMap`, which drops the scalar type metadata that the scalar typed path preserves in `FieldWithDataType`. A normal full projection of a shredded VARIANT with schema `metadata, value, typed_value: array<decimal(...)>` takes the row-wise path because `column_ids` is empty and the optional `value` leaf is selected, then decimal array elements are parsed back from JSON numbers instead of inserted with the original decimal precision/scale. That makes `SELECT v` produce a different VARIANT shape/type fidelity than the direct typed path for the same typed data. Please build the `VariantMap` for typed arrays without a JSON round trip (or otherwise preserve element type metadata) and add coverage for a typed array of DECIMAL or another non-JSON-native scalar under full VARIANT projection.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466927877
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3252996648
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -1001,6 +2427,143 @@ Status StructColumnReader::read_column_data(
return Status::OK();
}
+Status VariantColumnReader::init(io::FileReaderSPtr file, FieldSchema* field,
+ const tparquet::RowGroup& row_group, size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ RuntimeState* state, bool in_collection,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ _field_schema = field;
+ _column_ids = column_ids;
+ _variant_struct_field = std::make_unique<FieldSchema>(*field);
+
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field->children.size());
+ child_names.reserve(field->children.size());
+ for (const auto& child : field->children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ DataTypePtr variant_struct_type = std::make_shared<DataTypeStruct>(child_types, child_names);
+ if (field->data_type->is_nullable()) {
+ variant_struct_type = make_nullable(variant_struct_type);
+ }
Review Comment:
Wrapping the synthetic struct in `Nullable` makes optional top-level VARIANT columns fail reader creation. `VariantColumnReader::init()` passes `_variant_struct_field` back through `ParquetColumnReader::create()`, but that factory dispatches complex types using `field->data_type->get_primitive_type() == TYPE_STRUCT` before unwrapping nullable. For a valid schema like `optional group v (VARIANT) { required binary metadata; optional binary value; }`, this branch turns the internal struct into `Nullable(Struct(...))`, the factory skips the struct case and falls through to the scalar-reader path with the group field's `physical_column_index` (`-1`), so the scan can index `row_group.columns[-1]` or fail before VARIANT decoding. Please keep the synthetic struct non-nullable and carry root nullability separately, or make the factory unwrap nullable for complex-type dispatch, with coverage for an optional top-level VARIANT column.
##########
be/src/format/parquet/parquet_nested_column_utils.cpp:
##########
@@ -0,0 +1,459 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/parquet/parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <cctype>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "core/data_type/data_type_nullable.h"
+#include "format/parquet/schema_desc.h"
+
+namespace doris {
+namespace {
+
+enum class NestedPathMode {
+ NAME,
+ FIELD_ID,
+};
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema,
+ std::string_view name) {
+ for (const auto& child : field_schema.children) {
+ if (child.name == name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_structural_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+struct VariantColumnIdExtractionResult {
+ bool has_child_columns = false;
+ bool needs_metadata = false;
+};
+
+bool is_shredded_variant_field(const FieldSchema& field_schema) {
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field_schema.children) {
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_value) {
+ return true;
+ }
+ if (typed_value == nullptr) {
+ return false;
+ }
+ const auto type = remove_nullable(typed_value->data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+bool add_shredded_variant_field_value(const FieldSchema& shredded_field,
+ std::set<uint64_t>& column_ids) {
+ if (const auto* value = find_child_by_structural_name(shredded_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ return true;
+ }
+ return false;
+}
+
+bool is_variant_array_subscript(std::string_view path) {
+ return !path.empty() &&
+ std::all_of(path.begin(), path.end(), [](unsigned char c) { return std::isdigit(c); });
+}
+
+bool is_terminal_variant_meta_component(std::string_view path) {
+ return path == "NULL" || path == "OFFSET";
+}
+
+const std::vector<std::string>& effective_variant_path(const std::vector<std::string>& raw_path,
+ std::vector<std::string>& stripped_path) {
+ if (!raw_path.empty() && is_terminal_variant_meta_component(raw_path.back())) {
+ stripped_path.assign(raw_path.begin(), raw_path.end() - 1);
+ return stripped_path;
+ }
+ return raw_path;
+}
+
+bool contains_inherited_metadata_value(const FieldSchema& field_schema) {
+ if (is_shredded_variant_field(field_schema) &&
+ find_child_by_structural_name(field_schema, "value") != nullptr) {
+ return true;
+ }
+ return std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_inherited_metadata_value(child); });
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+VariantColumnIdExtractionResult extract_shredded_variant_field_ids(
+ const FieldSchema& shredded_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_structural_name(shredded_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& raw_path : paths) {
+ std::vector<std::string> stripped_path;
+ const auto& path = effective_variant_path(raw_path, stripped_path);
+ if (path.empty()) {
+ add_column_id_range(shredded_field, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata |= contains_inherited_metadata_value(shredded_field);
+ continue;
+ }
+
+ bool has_selected_columns = add_shredded_variant_field_value(shredded_field, column_ids);
+ result.needs_metadata |= has_selected_columns;
+ if (typed_value != nullptr) {
+ const auto typed_value_type = remove_nullable(typed_value->data_type);
+ if (typed_value_type->get_primitive_type() != TYPE_STRUCT) {
+ auto child_result =
+ extract_variant_typed_nested_column_ids(*typed_value, {path}, column_ids);
+ if (child_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ result.needs_metadata |= child_result.needs_metadata;
+ has_selected_columns = true;
+ }
+ } else if (const auto* typed_child = find_child_by_exact_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ result.needs_metadata |= contains_inherited_metadata_value(*typed_child);
+ column_ids.insert(typed_value->get_column_id());
+ has_selected_columns = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ auto child_result = extract_variant_typed_nested_column_ids(
+ *typed_child, child_paths, column_ids);
+ if (child_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ result.needs_metadata |= child_result.needs_metadata;
+ has_selected_columns = true;
+ }
+ }
+ }
+ }
+ result.has_child_columns |= has_selected_columns;
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(shredded_field.get_column_id());
+ }
+ return result;
+}
+
+VariantColumnIdExtractionResult extract_variant_nested_column_ids(
+ const FieldSchema& variant_field, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_structural_name(variant_field, "typed_value");
+ VariantColumnIdExtractionResult result;
+
+ for (const auto& raw_path : paths) {
+ std::vector<std::string> stripped_path;
+ const auto& path = effective_variant_path(raw_path, stripped_path);
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ result.has_child_columns = true;
+ continue;
+ }
+
+ VariantColumnIdExtractionResult typed_result;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_exact_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ typed_result.has_child_columns = true;
+ typed_result.needs_metadata = contains_inherited_metadata_value(*typed_child);
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ typed_result = extract_variant_typed_nested_column_ids(*typed_child,
+ child_paths, column_ids);
+ }
+ if (typed_result.has_child_columns) {
+ column_ids.insert(typed_value->get_column_id());
+ if (typed_result.needs_metadata) {
+ add_variant_metadata(variant_field, column_ids);
+ }
+ }
+ }
+ }
+
+ if (!typed_result.has_child_columns) {
+ add_variant_value(variant_field, column_ids);
+ }
+ result.has_child_columns = true;
+ }
+
+ if (result.has_child_columns) {
+ column_ids.insert(variant_field.get_column_id());
+ }
+ return result;
+}
+
+VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_VARIANT) {
+ return extract_variant_nested_column_ids(field_schema, paths, column_ids);
+ }
+ if (is_shredded_variant_field(field_schema)) {
+ return extract_shredded_variant_field_ids(field_schema, paths, column_ids);
+ }
+
+ VariantColumnIdExtractionResult result;
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>> child_paths_by_name;
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(field_schema, column_ids);
+ result.has_child_columns = true;
+ result.needs_metadata |= contains_inherited_metadata_value(field_schema);
+ continue;
+ }
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ const bool is_list =
+ field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ const std::string child_key =
+ is_list && is_variant_array_subscript(path[0]) ? "*" : path[0];
+ child_paths_by_name[child_key].push_back(std::move(remaining));
Review Comment:
This only maps explicit numeric VARIANT array subscripts to the LIST wildcard. FE can also produce element subpaths without a numeric component when the array element is exposed by `explode`, for example `SELECT x['metric'] FROM t LATERAL VIEW explode(v['arr']) tmp AS x`: `AccessPathPlanCollector` appends the generated slot suffix directly for VARIANT explode children, so BE receives a path like `[arr, metric]`. After entering the shredded `arr` field, this code recurses into the typed ARRAY with `[metric]`, groups it under `metric`, and the LIST child lookup below only searches for `*`; no typed leaf such as `v.typed_value.arr.typed_value.list.element.metric` is selected and typed-only shredded arrays fall back to top-level `v.value`/missing data. Please treat a non-subscript component under a VARIANT typed array as element access without consuming that component, and add Hive/Iceberg coverage for exploding a typed-only shredded VARIANT array and reading a field from the element.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468724882
<details>
<summary>TPC-H: <b>Total hot run time: 31274 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 0b5f089ab1c302d77f7a29632bfb66db8b919dcd, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17667 3822 3857 3822
q2 q3 10740 1338 785 785
q4 4687 476 343 343
q5 7546 2218 2124 2124
q6 253 175 144 144
q7 924 792 618 618
q8 9434 1622 1667 1622
q9 6693 4886 4842 4842
q10 6439 2131 1826 1826
q11 446 280 236 236
q12 692 423 286 286
q13 18325 3406 2781 2781
q14 269 252 237 237
q15 q16 814 767 705 705
q17 1017 908 930 908
q18 6903 5659 5664 5659
q19 1216 1214 1162 1162
q20 544 414 287 287
q21 6215 2653 2578 2578
q22 443 365 309 309
Total cold run time: 101267 ms
Total hot run time: 31274 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4627 4572 4689 4572
q2 q3 4865 5149 4579 4579
q4 2236 2213 1412 1412
q5 4832 4711 4671 4671
q6 230 177 135 135
q7 1892 1716 1527 1527
q8 2385 2004 1892 1892
q9 7175 7158 7203 7158
q10 4487 4445 4018 4018
q11 549 386 355 355
q12 711 721 500 500
q13 3004 3433 2830 2830
q14 281 280 243 243
q15 q16 695 706 609 609
q17 1251 1238 1238 1238
q18 7319 6812 6784 6784
q19 1133 1061 1147 1061
q20 2245 2224 1933 1933
q21 5324 4670 4491 4491
q22 517 455 420 420
Total cold run time: 55758 ms
Total hot run time: 50428 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467001062
<details>
<summary>TPC-H: <b>Total hot run time: 30562 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 87943d8bbf33c9afaaa06c5545e8214cdd477a19, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17719 3841 3828 3828
q2 q3 10737 1359 794 794
q4 4678 466 343 343
q5 7626 2224 2027 2027
q6 230 173 137 137
q7 917 757 646 646
q8 9349 1613 1503 1503
q9 6268 4881 4908 4881
q10 6321 2118 1753 1753
q11 436 277 248 248
q12 687 430 292 292
q13 18172 3788 2788 2788
q14 260 256 232 232
q15 q16 805 771 702 702
q17 990 925 876 876
q18 6832 5676 5531 5531
q19 1174 1243 1051 1051
q20 499 411 262 262
q21 5676 2525 2364 2364
q22 428 350 304 304
Total cold run time: 99804 ms
Total hot run time: 30562 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4192 4177 4126 4126
q2 q3 4482 4901 4376 4376
q4 2097 2226 1397 1397
q5 4385 4280 4293 4280
q6 226 172 129 129
q7 1757 2142 1699 1699
q8 2483 2113 2110 2110
q9 7832 7740 7722 7722
q10 4552 4451 4074 4074
q11 566 408 365 365
q12 712 851 595 595
q13 3320 3674 2996 2996
q14 299 302 283 283
q15 q16 727 734 683 683
q17 1360 1331 1352 1331
q18 7803 7220 7074 7074
q19 1146 1085 1092 1085
q20 2220 2204 1912 1912
q21 5376 4627 4469 4469
q22 508 459 398 398
Total cold run time: 56043 ms
Total hot run time: 51104 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467173582
<details>
<summary>TPC-DS: <b>Total hot run time: 170784 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 996c67ff18c29901aa8e182abd2ad228ed3b3611, data reload: false
query5 4359 660 521 521
query6 336 229 207 207
query7 4239 552 318 318
query8 336 241 230 230
query9 8853 4041 3999 3999
query10 445 342 294 294
query11 5798 2405 2271 2271
query12 189 131 127 127
query13 1286 599 459 459
query14 6027 5384 5055 5055
query14_1 4420 4408 4396 4396
query15 211 210 185 185
query16 977 451 438 438
query17 968 719 598 598
query18 2449 506 361 361
query19 236 213 172 172
query20 134 131 135 131
query21 221 140 118 118
query22 13662 13605 13392 13392
query23 17255 16320 15970 15970
query23_1 16208 16145 16134 16134
query24 7570 1781 1329 1329
query24_1 1321 1320 1295 1295
query25 577 505 455 455
query26 1328 321 178 178
query27 2671 553 347 347
query28 4478 1951 1923 1923
query29 1008 656 521 521
query30 311 242 196 196
query31 1133 1069 944 944
query32 98 80 75 75
query33 567 386 309 309
query34 1176 1179 629 629
query35 763 776 681 681
query36 1314 1360 1120 1120
query37 147 101 95 95
query38 3224 3139 3063 3063
query39 944 941 881 881
query39_1 869 886 879 879
query40 234 147 126 126
query41 66 65 62 62
query42 114 112 107 107
query43 333 332 286 286
query44
query45 211 202 193 193
query46 1131 1155 743 743
query47 2337 2270 2166 2166
query48 391 390 285 285
query49 633 488 386 386
query50 1020 342 252 252
query51 4310 4327 4172 4172
query52 101 107 95 95
query53 266 281 206 206
query54 303 274 268 268
query55 94 91 83 83
query56 298 305 302 302
query57 1394 1377 1293 1293
query58 289 271 272 271
query59 1587 1627 1420 1420
query60 317 331 312 312
query61 163 159 158 158
query62 668 634 564 564
query63 247 204 211 204
query64 2468 801 641 641
query65
query66 1719 486 391 391
query67 30127 29969 29922 29922
query68
query69 468 333 310 310
query70 974 954 990 954
query71 313 271 269 269
query72 2987 2706 2497 2497
query73 856 739 434 434
query74 5095 4906 4783 4783
query75 2675 2620 2273 2273
query76 2321 1143 763 763
query77 407 412 343 343
query78 12225 12122 11748 11748
query79 1439 1031 725 725
query80 663 559 453 453
query81 452 285 245 245
query82 1390 160 122 122
query83 371 277 251 251
query84 311 135 113 113
query85 895 567 466 466
query86 391 337 339 337
query87 3438 3386 3216 3216
query88 3509 2629 2652 2629
query89 443 387 348 348
query90 1998 181 184 181
query91 180 169 142 142
query92 83 76 76 76
query93 1536 1393 894 894
query94 543 343 314 314
query95 672 402 348 348
query96 1053 790 353 353
query97 2674 2680 2557 2557
query98 237 231 225 225
query99 1107 1121 1004 1004
Total cold run time: 253633 ms
Total hot run time: 170784 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467776167
<details>
<summary>TPC-DS: <b>Total hot run time: 169738 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 798ea04f582ea3a13d3ac44831549830facd1b2a, data reload: false
query5 4321 653 518 518
query6 316 220 202 202
query7 4221 586 291 291
query8 325 232 222 222
query9 8819 4077 4021 4021
query10 445 351 305 305
query11 5809 2396 2229 2229
query12 190 131 131 131
query13 1321 637 390 390
query14 6094 5294 4973 4973
query14_1 4309 4310 4288 4288
query15 215 204 188 188
query16 1008 451 432 432
query17 1135 702 577 577
query18 2568 470 342 342
query19 209 193 171 171
query20 140 131 130 130
query21 210 145 117 117
query22 13584 13601 13384 13384
query23 17140 16314 16033 16033
query23_1 16218 16157 16092 16092
query24 7490 1775 1309 1309
query24_1 1318 1328 1301 1301
query25 582 518 469 469
query26 1324 308 172 172
query27 2694 529 336 336
query28 4444 2001 1950 1950
query29 1025 648 522 522
query30 305 256 203 203
query31 1142 1070 956 956
query32 83 77 79 77
query33 534 360 304 304
query34 1165 1135 637 637
query35 773 798 680 680
query36 1349 1387 1148 1148
query37 157 114 91 91
query38 3200 3134 3038 3038
query39 939 967 886 886
query39_1 875 914 895 895
query40 240 150 132 132
query41 72 71 69 69
query42 113 112 114 112
query43 327 328 282 282
query44
query45 216 203 200 200
query46 1058 1190 742 742
query47 2303 2312 2227 2227
query48 422 441 296 296
query49 654 517 422 422
query50 1025 353 256 256
query51 4379 4395 4219 4219
query52 112 111 103 103
query53 266 291 220 220
query54 329 304 292 292
query55 99 97 91 91
query56 323 329 334 329
query57 1481 1440 1333 1333
query58 315 293 288 288
query59 1608 1735 1488 1488
query60 338 333 328 328
query61 193 183 206 183
query62 695 653 572 572
query63 245 200 209 200
query64 2385 818 636 636
query65
query66 1696 523 358 358
query67 30230 29981 29297 29297
query68
query69 452 335 310 310
query70 1023 1014 1000 1000
query71 299 278 274 274
query72 3006 2751 2629 2629
query73 862 766 420 420
query74 5071 4919 4759 4759
query75 2720 2598 2235 2235
query76 2277 1118 767 767
query77 403 396 332 332
query78 12163 12089 11525 11525
query79 1421 1021 722 722
query80 784 579 460 460
query81 481 287 239 239
query82 1371 164 122 122
query83 352 271 243 243
query84 266 141 106 106
query85 908 531 438 438
query86 403 313 325 313
query87 3440 3360 3213 3213
query88 3516 2668 2645 2645
query89 449 395 336 336
query90 1861 177 175 175
query91 176 170 142 142
query92 80 76 75 75
query93 1488 1630 885 885
query94 611 359 324 324
query95 655 494 354 354
query96 1057 768 312 312
query97 2721 2701 2578 2578
query98 231 233 228 228
query99 1110 1087 972 972
Total cold run time: 253675 ms
Total hot run time: 169738 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468271208
<details>
<summary>TPC-H: <b>Total hot run time: 30790 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 013742cd4f2c58ebb182cb3db9472867f4e5e860, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17858 3852 3824 3824
q2 q3 10758 1350 816 816
q4 4687 469 343 343
q5 7550 2230 2096 2096
q6 265 176 141 141
q7 938 787 624 624
q8 9390 1754 1558 1558
q9 6517 4914 4857 4857
q10 6431 2114 1785 1785
q11 461 277 257 257
q12 694 428 298 298
q13 18201 3383 2743 2743
q14 276 259 237 237
q15 q16 824 772 704 704
q17 992 932 1009 932
q18 6857 5692 5496 5496
q19 1305 1207 1045 1045
q20 525 409 269 269
q21 5889 2603 2449 2449
q22 438 357 316 316
Total cold run time: 100856 ms
Total hot run time: 30790 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4205 4266 4071 4071
q2 q3 4463 4881 4320 4320
q4 2120 2201 1380 1380
q5 4425 4227 4284 4227
q6 233 175 131 131
q7 2191 1887 1642 1642
q8 2499 2140 2055 2055
q9 7795 7777 7628 7628
q10 4534 4478 4054 4054
q11 575 410 462 410
q12 764 735 519 519
q13 3291 3674 3007 3007
q14 291 296 290 290
q15 q16 745 751 667 667
q17 1327 1322 1376 1322
q18 7925 7331 7058 7058
q19 1153 1117 1094 1094
q20 2200 2217 1934 1934
q21 5326 4637 4478 4478
q22 534 463 409 409
Total cold run time: 56596 ms
Total hot run time: 50696 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467387702
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253219373
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +127,1534 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static bool is_value_only_variant_wrapper_candidate(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_value = false;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "value" && child.physical_type == tparquet::Type::BYTE_ARRAY) {
+ has_value = true;
+ continue;
+ }
+ return false;
+ }
+ return has_value;
+}
+
+static Status get_binary_field(const Field& field, std::string* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ *value = field.get<TYPE_STRING>();
+ return Status::OK();
+ case TYPE_CHAR:
+ *value = field.get<TYPE_CHAR>();
+ return Status::OK();
+ case TYPE_VARCHAR:
+ *value = field.get<TYPE_VARCHAR>();
+ return Status::OK();
+ case TYPE_VARBINARY: {
+ auto ref = field.get<TYPE_VARBINARY>().to_string_ref();
+ value->assign(ref.data, ref.size);
+ return Status::OK();
+ }
+ default:
+ return Status::Corruption("Parquet VARIANT binary field has unexpected Doris type {}",
+ field.get_type_name());
+ }
+}
+
+static PathInData append_path(const PathInData& prefix, const PathInData& suffix) {
+ if (prefix.empty()) {
+ return suffix;
+ }
+ if (suffix.empty()) {
+ return prefix;
+ }
+ PathInDataBuilder builder;
+ builder.append(prefix.get_parts(), false);
+ builder.append(suffix.get_parts(), false);
+ return builder.build();
+}
+
+static Status make_jsonb_field(std::string_view json, FieldWithDataType* value) {
+ JsonBinaryValue jsonb_value;
+ RETURN_IF_ERROR(jsonb_value.from_json_string(json.data(), json.size()));
+ value->field =
+ Field::create_field<TYPE_JSONB>(JsonbField(jsonb_value.value(), jsonb_value.size()));
+ value->base_scalar_type_id = TYPE_JSONB;
+ value->num_dimensions = 0;
+ value->precision = 0;
+ value->scale = 0;
+ return Status::OK();
+}
+
+static std::string make_null_array_json(size_t elements) {
+ std::string json = "[";
+ for (size_t i = 0; i < elements; ++i) {
+ if (i != 0) {
+ json.push_back(',');
+ }
+ json.append("null");
+ }
+ json.push_back(']');
+ return json;
+}
+
+static Status make_empty_object_field(Field* field) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field("{}", &value));
+ *field = std::move(value.field);
+ return Status::OK();
+}
+
+static Status insert_jsonb_value(const PathInData& path, std::string_view json,
+ VariantMap* values) {
+ FieldWithDataType value;
+ RETURN_IF_ERROR(make_jsonb_field(json, &value));
+ (*values)[path] = std::move(value);
+ return Status::OK();
+}
+
+static Status insert_empty_object_marker(const PathInData& path, VariantMap* values) {
+ return insert_jsonb_value(path, "{}", values);
+}
+
+static bool is_empty_object_marker(const FieldWithDataType& value) {
+ if (value.field.get_type() != TYPE_JSONB) {
+ return false;
+ }
+ const auto& jsonb = value.field.get<TYPE_JSONB>();
+ const JsonbDocument* document = nullptr;
+ Status st =
+ JsonbDocument::checkAndCreateDocument(jsonb.get_value(), jsonb.get_size(), &document);
+ if (!st.ok() || document == nullptr || document->getValue() == nullptr ||
+ !document->getValue()->isObject()) {
+ return false;
+ }
+ return document->getValue()->unpack<ObjectVal>()->numElem() == 0;
+}
+
+static Status collect_empty_object_markers(const rapidjson::Value& value, PathInDataBuilder* path,
+ VariantMap* values) {
+ if (!value.IsObject()) {
+ return Status::OK();
+ }
+ if (value.MemberCount() == 0) {
+ return insert_empty_object_marker(path->build(), values);
+ }
+ for (auto it = value.MemberBegin(); it != value.MemberEnd(); ++it) {
+ if (it->value.IsObject()) {
+ path->append(std::string_view(it->name.GetString(), it->name.GetStringLength()), false);
+ RETURN_IF_ERROR(collect_empty_object_markers(it->value, path, values));
+ path->pop_back();
+ }
+ }
+ return Status::OK();
+}
+
+static Status add_empty_object_markers_from_json(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ if (json.find("{}") == std::string::npos) {
+ return Status::OK();
+ }
+ rapidjson::Document document;
+ document.Parse(json.data(), json.size());
+ if (document.HasParseError()) {
+ return Status::Corruption("Invalid Parquet VARIANT decoded JSON");
+ }
+ PathInDataBuilder path;
+ path.append(prefix.get_parts(), false);
+ return collect_empty_object_markers(document, &path, values);
+}
+
+static Status parse_json_to_variant_map(const std::string& json, const PathInData& prefix,
+ VariantMap* values) {
+ auto parsed_column = ColumnVariant::create(0, false);
+ ParseConfig parse_config;
+ StringRef json_ref(json.data(), json.size());
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_util::parse_json_to_variant(*parsed_column, json_ref, nullptr, parse_config));
+ Field parsed = (*parsed_column)[0];
+ if (!parsed.is_null()) {
+ auto& parsed_values = parsed.get<TYPE_VARIANT>();
+ for (auto& [path, value] : parsed_values) {
+ (*values)[append_path(prefix, path)] = std::move(value);
+ }
+ }
+ RETURN_IF_ERROR(add_empty_object_markers_from_json(json, prefix, values));
+ return Status::OK();
+}
+
+static Status variant_map_to_json(VariantMap values, std::string* json) {
+ auto variant_column = ColumnVariant::create(0, false);
+ RETURN_IF_CATCH_EXCEPTION(
+ variant_column->insert(Field::create_field<TYPE_VARIANT>(std::move(values))));
+ DataTypeSerDe::FormatOptions options;
+ variant_column->serialize_one_row_to_string(0, json, options);
+ return Status::OK();
+}
+
+static bool path_has_prefix(const PathInData& path, const PathInData& prefix) {
+ const auto& parts = path.get_parts();
+ const auto& prefix_parts = prefix.get_parts();
+ if (parts.size() < prefix_parts.size()) {
+ return false;
+ }
+ for (size_t i = 0; i < prefix_parts.size(); ++i) {
+ if (parts[i] != prefix_parts[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool has_descendant_path(const VariantMap& values, const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ return std::ranges::any_of(values, [&](const auto& entry) {
+ const auto& path = entry.first;
+ return path.get_parts().size() > prefix_size && path_has_prefix(path, prefix);
+ });
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* values,
+ const VariantMap& shadowing_values) {
+ for (auto it = values->begin(); it != values->end();) {
+ if (is_empty_object_marker(it->second) &&
+ (has_descendant_path(*values, it->first) ||
+ has_descendant_path(shadowing_values, it->first))) {
+ it = values->erase(it);
+ continue;
+ }
+ ++it;
+ }
+}
+
+static void erase_shadowed_empty_object_markers(VariantMap* value_values,
+ VariantMap* typed_values) {
+ erase_shadowed_empty_object_markers(value_values, *typed_values);
+ erase_shadowed_empty_object_markers(typed_values, *value_values);
+}
+
+static Status check_no_shredded_value_typed_duplicates(const VariantMap& value_values,
+ const VariantMap& typed_values,
+ const PathInData& prefix) {
+ const size_t prefix_size = prefix.get_parts().size();
+ for (const auto& value_entry : value_values) {
+ const auto& value_path = value_entry.first;
+ if (!path_has_prefix(value_path, prefix)) {
+ continue;
+ }
+ if (value_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(value_entry.second) &&
+ !has_descendant_path(typed_values, value_path)) {
+ continue;
+ }
+ if (!typed_values.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT residual value conflicts with typed_value at path {}",
+ value_path.get_path());
+ }
+ continue;
+ }
+ for (const auto& typed_entry : typed_values) {
+ const auto& typed_path = typed_entry.first;
+ if (!path_has_prefix(typed_path, prefix)) {
+ continue;
+ }
+ if (typed_path.get_parts().size() == prefix_size) {
+ if (is_empty_object_marker(typed_entry.second) &&
+ !has_descendant_path(value_values, typed_path)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ if (value_path.get_parts()[prefix_size] == typed_path.get_parts()[prefix_size]) {
+ if (value_path == typed_path && is_empty_object_marker(value_entry.second) &&
+ is_empty_object_marker(typed_entry.second)) {
+ continue;
+ }
+ return Status::Corruption(
+ "Parquet VARIANT residual value and typed_value contain duplicate field {}",
+ value_path.get_parts()[prefix_size].key);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+static bool has_direct_typed_parent_null(const std::vector<const NullMap*>& null_maps, size_t row) {
+ return std::ranges::any_of(null_maps, [&](const NullMap* null_map) {
+ DCHECK_LT(row, null_map->size());
+ return (*null_map)[row];
+ });
+}
+
+static void insert_direct_typed_leaf_range(const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps,
+ IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ nullable_leaf.get_nested_column().insert_range_from(*value_column, start, rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static bool is_temporal_variant_leaf_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_uuid_typed_value_field(const FieldSchema& field_schema);
+static bool contains_uuid_typed_value_field(const FieldSchema& field_schema);
+
+static DataTypePtr direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return std::make_shared<DataTypeInt64>();
+ }
+ return type;
+}
+
+static DataTypePtr direct_variant_leaf_type(const FieldSchema& field_schema) {
+ if (is_uuid_typed_value_field(field_schema)) {
+ return std::make_shared<DataTypeString>();
+ }
+ return direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool contains_temporal_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ if (is_temporal_variant_leaf_type(type->get_primitive_type())) {
+ return true;
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ return contains_temporal_variant_leaf_type(
+ assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return false;
+}
+
+static int64_t direct_temporal_variant_value(PrimitiveType type, const IColumn& column,
+ size_t row) {
+ switch (type) {
+ case TYPE_TIMEV2:
+ return static_cast<int64_t>(
+ std::llround(assert_cast<const ColumnTimeV2&>(column).get_data()[row]));
+ case TYPE_DATE:
+ return variant_date_value(assert_cast<const ColumnDate&>(column).get_data()[row]);
+ case TYPE_DATETIME:
+ return variant_datetime_value(assert_cast<const ColumnDateTime&>(column).get_data()[row]);
+ case TYPE_DATEV2:
+ return variant_date_value(assert_cast<const ColumnDateV2&>(column).get_data()[row]);
+ case TYPE_DATETIMEV2:
+ return variant_datetime_value(assert_cast<const ColumnDateTimeV2&>(column).get_data()[row]);
+ case TYPE_TIMESTAMPTZ:
+ return variant_datetime_value(
+ assert_cast<const ColumnTimeStampTz&>(column).get_data()[row]);
+ default:
+ DORIS_CHECK(false);
+ return 0;
+ }
+}
+
+static void insert_direct_typed_temporal_leaf_range(
+ PrimitiveType type, const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnInt64&>(nullable_leaf.get_nested_column()).get_data();
+ data.reserve(data.size() + rows);
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ data.push_back(direct_temporal_variant_value(type, *value_column, row));
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ null_map.push_back(leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row));
+ }
+}
+
+static Status insert_direct_typed_uuid_leaf_range(
+ const IColumn& column, size_t start, size_t rows,
+ const std::vector<const NullMap*>& parent_null_maps, IColumn* variant_leaf) {
+ auto& nullable_leaf = assert_cast<ColumnNullable&>(*variant_leaf);
+ const IColumn* value_column = &column;
+ const NullMap* leaf_null_map = nullptr;
+ if (const auto* nullable_column = check_and_get_column<ColumnNullable>(&column)) {
+ value_column = &nullable_column->get_nested_column();
+ leaf_null_map = &nullable_column->get_null_map_data();
+ }
+
+ auto& data = assert_cast<ColumnString&>(nullable_leaf.get_nested_column());
+ auto& null_map = nullable_leaf.get_null_map_data();
+ null_map.reserve(null_map.size() + rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const size_t row = start + i;
+ const bool leaf_is_null = leaf_null_map != nullptr && (*leaf_null_map)[row];
+ const bool is_null = leaf_is_null || has_direct_typed_parent_null(parent_null_maps, row);
+ if (is_null) {
+ data.insert_default();
+ null_map.push_back(1);
+ continue;
+ }
+ StringRef bytes = value_column->get_data_at(row);
+ if (bytes.size != 16) {
+ return Status::Corruption("Parquet VARIANT UUID typed_value has invalid length {}",
+ bytes.size);
+ }
+ std::string uuid =
+ parquet::format_variant_uuid(reinterpret_cast<const uint8_t*>(bytes.data));
+ data.insert_data(uuid.data(), uuid.size());
+ null_map.push_back(0);
+ }
+ return Status::OK();
+}
+
+static void append_json_string(std::string_view value, std::string* json) {
+ auto column = ColumnString::create();
+ VectorBufferWriter writer(*column);
+ writer.write_json_string(value);
+ writer.commit();
+ json->append(column->get_data_at(0).data, column->get_data_at(0).size);
+}
+
+static bool is_column_selected(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ return column_ids.empty() || column_ids.find(field_schema.get_column_id()) != column_ids.end();
+}
+
+static bool has_selected_column(const FieldSchema& field_schema,
+ const std::set<uint64_t>& column_ids) {
+ if (is_column_selected(field_schema, column_ids)) {
+ return true;
+ }
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_column(child, column_ids);
+ });
+}
+
+static bool is_direct_variant_leaf_type(const DataTypePtr& data_type) {
+ const auto& type = remove_nullable(data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY:
+ return true;
+ case TYPE_TIMEV2:
+ case TYPE_DATE:
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMPTZ:
+ return true;
+ case TYPE_ARRAY: {
+ const auto* array_type = assert_cast<const DataTypeArray*>(type.get());
+ return !contains_temporal_variant_leaf_type(array_type->get_nested_type()) &&
+ is_direct_variant_leaf_type(array_type->get_nested_type());
+ }
+ default:
+ return false;
+ }
+}
+
+static bool can_direct_read_typed_value(const FieldSchema& field_schema, bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return true;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int value_idx = find_child_idx(field_schema, "value");
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ return (value_idx < 0 ||
+ !has_selected_column(field_schema.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ can_direct_read_typed_value(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::all_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return can_direct_read_typed_value(child, true, column_ids);
+ });
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY && contains_uuid_typed_value_field(field_schema)) {
+ return false;
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool has_selected_direct_typed_leaf(const FieldSchema& field_schema,
+ bool allow_variant_wrapper,
+ const std::set<uint64_t>& column_ids) {
+ if (!has_selected_column(field_schema, column_ids)) {
+ return false;
+ }
+ if (allow_variant_wrapper && is_variant_wrapper_field(field_schema, false)) {
+ const int typed_value_idx = find_child_idx(field_schema, "typed_value");
+ DCHECK_GE(typed_value_idx, 0);
+ return has_selected_direct_typed_leaf(field_schema.children[typed_value_idx], false,
+ column_ids);
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_STRUCT) {
+ return std::any_of(field_schema.children.begin(), field_schema.children.end(),
+ [&column_ids](const FieldSchema& child) {
+ return has_selected_direct_typed_leaf(child, true, column_ids);
+ });
+ }
+ if (type->get_primitive_type() == TYPE_ARRAY && contains_uuid_typed_value_field(field_schema)) {
+ return false;
+ }
+ return is_direct_variant_leaf_type(field_schema.data_type);
+}
+
+static bool can_use_direct_typed_only_value(const FieldSchema& variant_field,
+ const std::set<uint64_t>& column_ids) {
+ const int value_idx = find_child_idx(variant_field, "value");
+ const int typed_value_idx = find_child_idx(variant_field, "typed_value");
+ return (value_idx < 0 || !has_selected_column(variant_field.children[value_idx], column_ids)) &&
+ typed_value_idx >= 0 &&
+ has_selected_direct_typed_leaf(variant_field.children[typed_value_idx], false,
+ column_ids) &&
+ can_direct_read_typed_value(variant_field.children[typed_value_idx], false, column_ids);
+}
+
+static DataTypePtr make_variant_struct_reader_type(const FieldSchema& field) {
+ DataTypes child_types;
+ Strings child_names;
+ child_types.reserve(field.children.size());
+ child_names.reserve(field.children.size());
+ for (const auto& child : field.children) {
+ child_types.push_back(make_nullable(child.data_type));
+ child_names.push_back(child.name);
+ }
+ return std::make_shared<DataTypeStruct>(child_types, child_names);
+}
+
+static ColumnPtr make_variant_struct_read_column(const FieldSchema& field,
+ const DataTypePtr& variant_struct_type) {
+ if (field.data_type->is_nullable()) {
+ return make_nullable(variant_struct_type)->create_column();
+ }
+ return variant_struct_type->create_column();
+}
+
+static void fill_variant_field_info(FieldWithDataType* value) {
+ FieldInfo info;
+ variant_util::get_field_info(value->field, &info);
+ DCHECK_LE(info.num_dimensions, std::numeric_limits<uint8_t>::max());
+ value->base_scalar_type_id = info.scalar_type_id;
+ value->num_dimensions = static_cast<uint8_t>(info.num_dimensions);
+}
+
+static void fill_variant_leaf_type_info(const DataTypePtr& data_type, FieldWithDataType* value) {
+ auto leaf_type = remove_nullable(data_type);
+ while (leaf_type->get_primitive_type() == TYPE_ARRAY) {
+ leaf_type = remove_nullable(
+ assert_cast<const DataTypeArray*>(leaf_type.get())->get_nested_type());
+ }
+ if (is_decimal(leaf_type->get_primitive_type())) {
+ value->precision = leaf_type->get_precision();
+ value->scale = leaf_type->get_scale();
+ }
+}
+
+template <PrimitiveType Primitive>
+static Status fill_floating_point_variant_field(const Field& field, FieldWithDataType* value) {
+ const auto typed_value = field.get<Primitive>();
+ if (!std::isfinite(typed_value)) {
+ return Status::NotSupported(
+ "Parquet VARIANT non-finite floating point typed_value is not supported");
+ }
+ value->field = field;
+ fill_variant_field_info(value);
+ return Status::OK();
+}
+
+static bool is_uuid_typed_value_field(const FieldSchema& field_schema) {
+ return field_schema.parquet_schema.__isset.logicalType &&
+ field_schema.parquet_schema.logicalType.__isset.UUID;
+}
+
+static bool contains_uuid_typed_value_field(const FieldSchema& field_schema) {
+ return is_uuid_typed_value_field(field_schema) ||
+ std::any_of(
+ field_schema.children.begin(), field_schema.children.end(),
+ [](const FieldSchema& child) { return contains_uuid_typed_value_field(child); });
+}
+
+static Status uuid_field_to_string(const Field& field, std::string* uuid) {
+ StringRef bytes;
+ switch (field.get_type()) {
+ case TYPE_STRING:
+ bytes = StringRef(field.get<TYPE_STRING>());
+ break;
+ case TYPE_CHAR:
+ bytes = StringRef(field.get<TYPE_CHAR>());
+ break;
+ case TYPE_VARCHAR:
+ bytes = StringRef(field.get<TYPE_VARCHAR>());
+ break;
+ case TYPE_VARBINARY:
+ bytes = field.get<TYPE_VARBINARY>().to_string_ref();
+ break;
+ default:
+ return Status::Corruption("Parquet VARIANT UUID typed_value has unexpected Doris type {}",
+ field.get_type_name());
+ }
+ if (bytes.size != 16) {
+ return Status::Corruption("Parquet VARIANT UUID typed_value has invalid length {}",
+ bytes.size);
+ }
+ *uuid = parquet::format_variant_uuid(reinterpret_cast<const uint8_t*>(bytes.data));
+ return Status::OK();
+}
+
+static Status fill_uuid_variant_field(const Field& field, FieldWithDataType* value) {
+ std::string uuid;
+ RETURN_IF_ERROR(uuid_field_to_string(field, &uuid));
+ value->field = Field::create_field<TYPE_STRING>(std::move(uuid));
+ value->base_scalar_type_id = TYPE_STRING;
+ return Status::OK();
+}
+
+static uint8_t direct_array_dimensions(const DataTypePtr& data_type) {
+ uint8_t num_dimensions = 0;
+ auto type = remove_nullable(data_type);
+ while (type->get_primitive_type() == TYPE_ARRAY) {
+ ++num_dimensions;
+ type = remove_nullable(assert_cast<const DataTypeArray*>(type.get())->get_nested_type());
+ }
+ return num_dimensions;
+}
+
+static Status convert_uuid_array_value(const FieldSchema& field_schema, const Field& field,
+ Field* converted) {
+ if (field.is_null()) {
+ *converted = Field();
+ return Status::OK();
+ }
+
+ const auto& type = remove_nullable(field_schema.data_type);
+ if (type->get_primitive_type() == TYPE_ARRAY) {
+ if (field_schema.children.empty()) {
+ return Status::Corruption(
+ "Parquet VARIANT UUID array typed_value has no element schema");
+ }
+ Array converted_elements;
+ const auto& elements = field.get<TYPE_ARRAY>();
+ converted_elements.reserve(elements.size());
+ for (const auto& element : elements) {
+ Field converted_element;
+ RETURN_IF_ERROR(convert_uuid_array_value(field_schema.children[0], element,
+ &converted_element));
+ converted_elements.push_back(std::move(converted_element));
+ }
+ *converted = Field::create_field<TYPE_ARRAY>(std::move(converted_elements));
+ return Status::OK();
+ }
+
+ if (!is_uuid_typed_value_field(field_schema)) {
+ return Status::Corruption("Parquet VARIANT UUID array has non-UUID element schema");
+ }
+ FieldWithDataType value;
+ RETURN_IF_ERROR(fill_uuid_variant_field(field, &value));
+ *converted = std::move(value.field);
+ return Status::OK();
+}
+
+static Status fill_uuid_array_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ RETURN_IF_ERROR(convert_uuid_array_value(field_schema, field, &value->field));
+ value->base_scalar_type_id = TYPE_STRING;
+ value->num_dimensions = direct_array_dimensions(field_schema.data_type);
+ return Status::OK();
+}
+
+static Status field_to_variant_field(const FieldSchema& field_schema, const Field& field,
+ FieldWithDataType* value, bool* present) {
+ if (field.is_null()) {
+ *present = false;
+ return Status::OK();
+ }
+ *present = true;
+ if (is_uuid_typed_value_field(field_schema)) {
+ return fill_uuid_variant_field(field, value);
+ }
+ const DataTypePtr& type = remove_nullable(field_schema.data_type);
+ switch (type->get_primitive_type()) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_LARGEINT:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_VARBINARY:
+ case TYPE_ARRAY:
+ value->field = field;
+ fill_variant_field_info(value);
+ fill_variant_leaf_type_info(type, value);
Review Comment:
This direct typed-array branch derives `base_scalar_type_id` and dimensions from the row value before applying schema metadata. For a valid typed leaf such as `typed_value: array<decimal(18,2)>`, rows containing `[]` or `[null]` have no non-null element, so `variant_util::get_field_info()` leaves the scalar type as `INVALID_TYPE`/`Nothing`; `fill_variant_leaf_type_info()` only fills precision/scale and never restores the array element type or dimensions from `field_schema.data_type`. Those rows are then inserted with different/invalid VARIANT type metadata from non-empty rows, losing the Parquet typed-array fidelity. This is distinct from the existing decimal-array JSON round-trip thread because it happens in the direct typed leaf path even without serializing through JSON. Please fill the array scalar type/dimensions from the typed schema when field-derived info is invalid, and add coverage for empty and all-null typed decimal arrays.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253434384
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java:
##########
@@ -391,6 +411,18 @@ private void collectByExpressions(Plan plan, StatementContext context, boolean b
}
}
+ private boolean collectWholeVariantOutput(Slot slot) {
+ if (!slot.getDataType().isVariantType()
+ || (slot instanceof SlotReference && ((SlotReference) slot).hasSubColPath())) {
+ return false;
+ }
+ List<String> path = new ArrayList<>();
+ path.add(slot.getName());
+ allSlotToAccessPaths.put(slot.getExprId().asInt(),
+ new CollectAccessPathResult(path, false, ColumnAccessPathType.DATA));
Review Comment:
Recording the full VARIANT output here is not propagated correctly through `LogicalGenerate` when the output slot comes from `explode(v)`. For example, `SELECT x FROM variant_tbl LATERAL VIEW explode(v) tmp AS x WHERE x['k'] IS NOT NULL` records a root path `[x]` for the project and a subpath `[x, k]` for the filter. In `visitLogicalGenerate()`, the root path enters the VARIANT child branch with `path.subList(1, path.size())`, which is an empty suffix; `visitSlotReference()` then sees the input `v` with an empty builder and records no root `[v]` access path. The filter still contributes `[v, k]`, so the scan can prune `v` to only `k` while the projected `x` requires the full exploded VARIANT value. This is distinct from the existing full-value expression and dynamic-subscript threads because the full-root requirement is introduced by generator-output propagation. Please make the `[x]` generator demand force a root access on the generator input for `explode(v)`, and add coverage co
mbining full `x` projection with a nested `x[...]` or `v[...]` predicate.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468541550
<details>
<summary>TPC-DS: <b>Total hot run time: 168536 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 2ade3cc902f640146304d1a4842bee639e90639b, data reload: false
query5 4357 651 517 517
query6 336 228 201 201
query7 4301 558 302 302
query8 319 226 213 213
query9 8826 3965 3951 3951
query10 452 337 287 287
query11 5758 2358 2153 2153
query12 188 128 125 125
query13 1276 630 404 404
query14 5917 5386 4994 4994
query14_1 4342 4339 4327 4327
query15 209 212 181 181
query16 966 449 396 396
query17 1120 716 585 585
query18 2426 481 347 347
query19 218 192 161 161
query20 133 139 127 127
query21 220 139 116 116
query22 13605 13520 13487 13487
query23 17215 16327 15944 15944
query23_1 16190 16052 16260 16052
query24 7443 1762 1297 1297
query24_1 1311 1279 1304 1279
query25 562 480 429 429
query26 1290 331 168 168
query27 2708 548 353 353
query28 4468 1928 1930 1928
query29 960 682 491 491
query30 326 248 193 193
query31 1136 1050 943 943
query32 83 75 70 70
query33 531 351 298 298
query34 1152 1159 656 656
query35 779 780 669 669
query36 1338 1349 1180 1180
query37 152 104 89 89
query38 3203 3168 3090 3090
query39 939 929 919 919
query39_1 864 877 864 864
query40 243 146 122 122
query41 66 64 62 62
query42 109 107 105 105
query43 326 335 290 290
query44
query45 209 207 194 194
query46 1086 1171 718 718
query47 2341 2349 2254 2254
query48 392 386 292 292
query49 641 488 385 385
query50 976 357 242 242
query51 4274 4300 4453 4300
query52 106 103 93 93
query53 254 279 209 209
query54 313 276 270 270
query55 94 89 85 85
query56 301 316 301 301
query57 1412 1400 1324 1324
query58 325 270 259 259
query59 1515 1612 1387 1387
query60 316 314 301 301
query61 152 160 152 152
query62 672 636 568 568
query63 251 201 211 201
query64 2470 862 696 696
query65
query66 1766 532 350 350
query67 29909 29915 29749 29749
query68
query69 467 331 306 306
query70 1018 983 980 980
query71 304 271 262 262
query72 2995 2614 2350 2350
query73 808 763 437 437
query74 5068 4911 4728 4728
query75 2665 2595 2293 2293
query76 2267 1107 780 780
query77 404 397 327 327
query78 12147 12126 11661 11661
query79 1461 1005 775 775
query80 650 543 445 445
query81 452 279 241 241
query82 1365 155 118 118
query83 343 279 242 242
query84 265 141 110 110
query85 894 536 452 452
query86 395 349 323 323
query87 3385 3379 3216 3216
query88 3477 2637 2583 2583
query89 440 386 339 339
query90 1903 179 185 179
query91 176 172 139 139
query92 84 80 84 80
query93 1515 1426 859 859
query94 552 356 305 305
query95 680 377 430 377
query96 1006 787 314 314
query97 2730 2706 2552 2552
query98 232 227 221 221
query99 1089 1104 976 976
Total cold run time: 252406 ms
Total hot run time: 168536 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466799551
<details>
<summary>TPC-DS: <b>Total hot run time: 169122 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 572b1f4924de30a96db26a600811d7cc9acafcf4, data reload: false
query5 4305 654 516 516
query6 333 233 237 233
query7 4286 532 304 304
query8 321 241 228 228
query9 8867 4009 3988 3988
query10 428 333 306 306
query11 5805 2402 2219 2219
query12 176 131 129 129
query13 1299 594 450 450
query14 5977 5355 5046 5046
query14_1 4394 4333 4296 4296
query15 207 202 184 184
query16 1046 464 414 414
query17 1161 716 588 588
query18 2463 491 354 354
query19 209 200 167 167
query20 137 130 131 130
query21 216 146 120 120
query22 13644 13523 13389 13389
query23 17169 16346 16117 16117
query23_1 16175 16141 16068 16068
query24 7475 1759 1280 1280
query24_1 1297 1304 1283 1283
query25 544 489 409 409
query26 1311 326 169 169
query27 2750 588 335 335
query28 4494 1979 1952 1952
query29 989 611 483 483
query30 305 238 203 203
query31 1101 1054 933 933
query32 88 76 71 71
query33 509 393 309 309
query34 1173 1158 628 628
query35 751 780 687 687
query36 1349 1362 1127 1127
query37 154 113 91 91
query38 3229 3151 3046 3046
query39 921 935 891 891
query39_1 886 874 881 874
query40 233 145 124 124
query41 65 63 61 61
query42 112 110 108 108
query43 325 319 284 284
query44
query45 211 199 196 196
query46 1021 1159 735 735
query47 2330 2332 2183 2183
query48 394 443 290 290
query49 628 490 391 391
query50 1060 346 253 253
query51 4301 4266 4228 4228
query52 103 109 94 94
query53 259 282 208 208
query54 310 263 250 250
query55 95 89 89 89
query56 313 314 300 300
query57 1425 1393 1308 1308
query58 296 272 265 265
query59 1558 1640 1395 1395
query60 321 336 322 322
query61 189 186 184 184
query62 670 642 568 568
query63 256 206 213 206
query64 2469 862 705 705
query65
query66 1724 511 372 372
query67 29885 29862 29780 29780
query68
query69 477 345 321 321
query70 1010 1010 957 957
query71 318 284 272 272
query72 3113 2660 2378 2378
query73 869 755 403 403
query74 5067 4891 4726 4726
query75 2630 2579 2253 2253
query76 2327 1141 758 758
query77 392 396 328 328
query78 12273 12140 11582 11582
query79 1247 955 737 737
query80 585 561 451 451
query81 455 274 243 243
query82 242 160 125 125
query83 273 271 249 249
query84 260 142 111 111
query85 883 524 453 453
query86 358 353 304 304
query87 3380 3430 3196 3196
query88 3492 2673 2655 2655
query89 427 391 334 334
query90 2175 177 175 175
query91 176 175 139 139
query92 76 78 69 69
query93 1414 1478 882 882
query94 525 373 319 319
query95 669 378 352 352
query96 1061 796 340 340
query97 2689 2711 2560 2560
query98 233 231 236 231
query99 1121 1093 988 988
Total cold run time: 251658 ms
Total hot run time: 169122 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466906054
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466906055
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467747074
<details>
<summary>TPC-H: <b>Total hot run time: 30727 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 798ea04f582ea3a13d3ac44831549830facd1b2a, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17724 3830 3786 3786
q2 q3 10761 1376 856 856
q4 4684 477 349 349
q5 7668 2203 2101 2101
q6 394 175 137 137
q7 931 761 635 635
q8 9396 1672 1497 1497
q9 6944 4925 4862 4862
q10 6431 2121 1809 1809
q11 423 267 243 243
q12 688 411 290 290
q13 18261 3467 2777 2777
q14 261 255 232 232
q15 q16 816 768 702 702
q17 914 890 874 874
q18 6780 5730 5478 5478
q19 1170 1192 1065 1065
q20 487 402 266 266
q21 5702 2560 2462 2462
q22 436 358 306 306
Total cold run time: 100871 ms
Total hot run time: 30727 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4153 4081 4070 4070
q2 q3 4564 4875 4249 4249
q4 2081 2208 1368 1368
q5 4379 4247 4260 4247
q6 219 178 128 128
q7 2074 1744 1695 1695
q8 2569 2107 2004 2004
q9 7673 7694 7578 7578
q10 4504 4441 4093 4093
q11 585 411 397 397
q12 722 726 508 508
q13 3448 3661 3008 3008
q14 305 293 272 272
q15 q16 773 724 632 632
q17 1318 1295 1284 1284
q18 7947 7333 7151 7151
q19 1109 1100 1111 1100
q20 2214 2214 2000 2000
q21 5272 4574 4426 4426
q22 517 467 428 428
Total cold run time: 56426 ms
Total hot run time: 50638 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253487495
##########
be/src/format/parquet/vparquet_column_reader.cpp:
##########
@@ -103,6 +127,1608 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset
}
}
+static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+static constexpr int64_t MICROS_PER_SECOND = 1000000;
+
+static int64_t variant_date_value(const VecDateTimeValue& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_date_value(const DateV2Value<DateV2ValueType>& value) {
+ return value.daynr() - UNIX_EPOCH_DAYNR;
+}
+
+static int64_t variant_datetime_value(const VecDateTimeValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND;
+}
+
+static int64_t variant_datetime_value(const DateV2Value<DateTimeV2ValueType>& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int64_t variant_datetime_value(const TimestampTzValue& value) {
+ int64_t timestamp = 0;
+ value.unix_timestamp(×tamp, cctz::utc_time_zone());
+ return timestamp * MICROS_PER_SECOND + value.microsecond();
+}
+
+static int find_child_idx(const FieldSchema& field, std::string_view name) {
+ for (int i = 0; i < field.children.size(); ++i) {
+ if (field.children[i].lower_case_name == name) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+static bool is_variant_wrapper_typed_value_child(const FieldSchema& field) {
+ auto type = remove_nullable(field.data_type);
+ return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY;
+}
+
+static bool is_variant_wrapper_field(const FieldSchema& field,
+ bool allow_scalar_typed_value_only_wrapper) {
+ auto type = remove_nullable(field.data_type);
+ if (type->get_primitive_type() != TYPE_STRUCT && type->get_primitive_type() != TYPE_VARIANT) {
+ return false;
+ }
+
+ bool has_metadata = false;
+ bool has_value = false;
+ const FieldSchema* typed_value = nullptr;
+ for (const auto& child : field.children) {
+ if (child.lower_case_name == "metadata") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_metadata = true;
+ continue;
+ }
+ if (child.lower_case_name == "value") {
+ if (child.physical_type != tparquet::Type::BYTE_ARRAY) {
+ return false;
+ }
+ has_value = true;
+ continue;
+ }
+ if (child.lower_case_name == "typed_value") {
+ typed_value = &child;
+ continue;
+ }
+ return false;
+ }
+ if (has_metadata && has_value) {
+ return type->get_primitive_type() == TYPE_VARIANT || typed_value != nullptr;
+ }
+ if (has_value) {
+ return typed_value != nullptr;
+ }
+ return typed_value != nullptr && (allow_scalar_typed_value_only_wrapper ||
+ is_variant_wrapper_typed_value_child(*typed_value));
+}
+
+static bool is_value_only_variant_wrapper_candidate(const FieldSchema& field) {
Review Comment:
This nullable guard also filters out valid value-only shredded residual wrappers. A shredded field under `typed_value` is typically optional to represent the field being absent; for a layout like `v.typed_value.metric { optional binary value }`, `metric.data_type` is `Nullable(Struct<value>)`. Row-wise reconstruction then skips the residual-wrapper path here and falls through to `typed_value_to_variant_map()`, which treats the bytes as an ordinary `metric.value` typed binary child instead of decoding the residual VARIANT payload with the inherited metadata. The new `RowWiseReadsValueOnlyNestedResidualField` test does not catch this because it builds `metric_field.data_type` as a non-nullable `DataTypeStruct`, while the pruning helper tests build the same residual field as nullable. Please distinguish user `value` structs from real shredded residual wrappers without rejecting optional wrapper groups, and add coverage with a nullable value-only residual field.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253598988
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java:
##########
@@ -98,14 +98,33 @@ public void collect(Expression expression) {
expression.accept(this, new CollectorContext(statementContext, bottomPredicate));
}
+ public void collectWholeVariantExpression(Expression expression) {
+ CollectorContext context = new CollectorContext(statementContext, bottomPredicate);
+ context.setCollectVariantRoot(true);
+ expression.accept(this, context);
+ }
+
private Void continueCollectAccessPath(Expression expr, CollectorContext context) {
return expr.accept(this, context);
}
+ private void recordVariantRootAccessPath(SlotReference slotReference, CollectorContext context) {
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId,
+ new CollectAccessPathResult(
+ ImmutableList.of(slotReference.getName()),
+ context.bottomFilter, ColumnAccessPathType.DATA));
+ }
+
@Override
public Void visit(Expression expr, CollectorContext context) {
for (Expression child : expr.children()) {
- child.accept(this, new CollectorContext(context.statementContext, context.bottomFilter));
+ if (child.getDataType().isVariantType()
Review Comment:
This propagation also turns top-level nullable VARIANT null checks into a fake nested subpath on OLAP scans. For `SELECT 1 FROM olap_variant_tbl WHERE v IS NULL`, `visitIsNull()` seeds the builder with `NULL`; this branch preserves that non-empty builder into the VARIANT child, and `visitSlotReference()` records `[v, NULL]` as a VARIANT DATA path. `NestedColumnPruning` handles VARIANT slots outside the `DataTypeAccessTree` null-only logic, and `visitLogicalOlapScan()` does not call `normalizeDataSkippingOnlyAccessPaths()`, so BE receives `NULL` as if it were a VARIANT field name rather than a top-level null-map/root requirement. File/TVF scans normalize this pseudo suffix away, which is why this is a distinct OLAP path issue. Please special-case VARIANT `IS NULL`/`IS NOT NULL` to record a valid root/null access for OLAP (or normalize these pseudo suffixes before building VARIANT `AccessPathInfo`) and add coverage for nullable top-level VARIANT null predicates.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468400719
<details>
<summary>TPC-H: <b>Total hot run time: 30862 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 1fe5df123ff286dc34a2f9ca8fef57c187bb95c4, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17622 3820 3825 3820
q2 q3 10775 1352 814 814
q4 4701 474 350 350
q5 7541 2263 2094 2094
q6 227 174 135 135
q7 944 770 642 642
q8 9343 1669 1560 1560
q9 5189 4873 4864 4864
q10 6383 2065 1796 1796
q11 452 273 248 248
q12 632 436 292 292
q13 18181 3439 2764 2764
q14 260 258 243 243
q15 q16 837 784 708 708
q17 1012 993 933 933
q18 6777 5669 5481 5481
q19 1465 1262 1080 1080
q20 522 391 275 275
q21 5875 2653 2455 2455
q22 443 358 308 308
Total cold run time: 99181 ms
Total hot run time: 30862 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4159 4059 4058 4058
q2 q3 4441 4921 4277 4277
q4 2138 2230 1370 1370
q5 4385 4227 4234 4227
q6 224 172 129 129
q7 1752 1939 1823 1823
q8 2845 2147 2342 2147
q9 7915 8103 7553 7553
q10 4599 4441 4013 4013
q11 558 409 412 409
q12 735 746 510 510
q13 3386 3645 2940 2940
q14 295 309 274 274
q15 q16 735 761 649 649
q17 1307 1312 1264 1264
q18 7512 6735 6707 6707
q19 1074 1098 1092 1092
q20 2221 2215 1923 1923
q21 5233 4631 4396 4396
q22 523 462 409 409
Total cold run time: 56037 ms
Total hot run time: 50170 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468693401
<details>
<summary>TPC-DS: <b>Total hot run time: 168746 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 89ddf16178f628f631b3352d572743a9b22f13ae, data reload: false
query5 4368 651 502 502
query6 340 213 190 190
query7 4215 549 299 299
query8 333 231 210 210
query9 8808 4026 4008 4008
query10 464 326 287 287
query11 5786 2341 2206 2206
query12 178 129 124 124
query13 1323 612 405 405
query14 5957 5368 5019 5019
query14_1 4342 4344 4301 4301
query15 211 199 182 182
query16 998 466 429 429
query17 1044 722 578 578
query18 2435 474 340 340
query19 209 204 160 160
query20 150 133 129 129
query21 213 140 117 117
query22 13566 13510 13381 13381
query23 17144 16278 15982 15982
query23_1 16202 16291 16409 16291
query24 7434 1822 1337 1337
query24_1 1337 1342 1314 1314
query25 587 509 452 452
query26 1340 328 183 183
query27 2657 589 344 344
query28 4966 1976 1940 1940
query29 1003 646 504 504
query30 303 238 205 205
query31 1102 1058 935 935
query32 99 81 78 78
query33 562 365 310 310
query34 1179 1076 664 664
query35 767 785 678 678
query36 1349 1350 1133 1133
query37 160 107 92 92
query38 3204 3150 3034 3034
query39 942 923 900 900
query39_1 902 895 875 875
query40 235 150 134 134
query41 72 70 74 70
query42 116 111 111 111
query43 329 331 284 284
query44
query45 214 210 199 199
query46 1050 1252 723 723
query47 2349 2341 2131 2131
query48 402 422 296 296
query49 650 500 397 397
query50 972 346 253 253
query51 4352 4338 4280 4280
query52 143 104 96 96
query53 252 290 203 203
query54 306 274 255 255
query55 94 89 82 82
query56 293 317 321 317
query57 1400 1407 1316 1316
query58 294 278 269 269
query59 1557 1666 1387 1387
query60 337 317 298 298
query61 156 155 153 153
query62 664 626 561 561
query63 241 202 205 202
query64 2426 807 644 644
query65
query66 1734 469 350 350
query67 30016 29988 29824 29824
query68
query69 466 334 296 296
query70 1018 972 933 933
query71 305 275 270 270
query72 2984 2692 2427 2427
query73 832 767 446 446
query74 5058 4958 4732 4732
query75 2673 2589 2263 2263
query76 2272 1125 742 742
query77 395 400 336 336
query78 12068 12035 11600 11600
query79 1423 1026 706 706
query80 652 542 448 448
query81 448 279 256 256
query82 1367 157 120 120
query83 363 274 248 248
query84 292 139 109 109
query85 871 544 454 454
query86 384 332 313 313
query87 3402 3397 3176 3176
query88 3529 2609 2636 2609
query89 441 390 336 336
query90 1944 180 179 179
query91 172 171 143 143
query92 87 81 73 73
query93 1489 1407 863 863
query94 534 341 327 327
query95 686 463 345 345
query96 1053 734 343 343
query97 2686 2718 2547 2547
query98 236 230 226 226
query99 1134 1067 982 982
Total cold run time: 253084 ms
Total hot run time: 168746 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468984045
<details>
<summary>TPC-DS: <b>Total hot run time: 168380 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 6b22fa723d69d717244b1cd0c3894901ce9a5b9b, data reload: false
query5 4329 652 508 508
query6 340 227 203 203
query7 4275 540 274 274
query8 331 226 215 215
query9 8841 3985 3965 3965
query10 459 341 305 305
query11 5848 2429 2265 2265
query12 185 133 125 125
query13 1294 597 422 422
query14 5990 5327 5044 5044
query14_1 4361 4353 4359 4353
query15 215 208 184 184
query16 998 472 411 411
query17 1185 749 607 607
query18 2559 492 366 366
query19 216 204 177 177
query20 138 137 131 131
query21 214 137 132 132
query22 13595 13542 13377 13377
query23 17143 16304 16037 16037
query23_1 16163 16151 16187 16151
query24 7662 1764 1289 1289
query24_1 1321 1323 1303 1303
query25 522 479 400 400
query26 1301 321 170 170
query27 2724 563 351 351
query28 4437 1955 1939 1939
query29 956 602 487 487
query30 296 242 198 198
query31 1113 1061 942 942
query32 87 75 74 74
query33 536 357 291 291
query34 1174 1129 633 633
query35 760 781 680 680
query36 1335 1387 1228 1228
query37 149 104 99 99
query38 3223 3136 3063 3063
query39 936 938 902 902
query39_1 874 865 879 865
query40 242 146 128 128
query41 79 62 62 62
query42 111 109 109 109
query43 320 322 277 277
query44
query45 215 197 196 196
query46 1065 1209 722 722
query47 2369 2360 2305 2305
query48 402 401 303 303
query49 637 526 385 385
query50 999 332 249 249
query51 4352 4181 4267 4181
query52 108 103 92 92
query53 251 288 207 207
query54 315 267 248 248
query55 90 90 84 84
query56 289 296 305 296
query57 1490 1398 1339 1339
query58 302 276 268 268
query59 1551 1698 1444 1444
query60 321 328 311 311
query61 160 152 151 151
query62 681 622 565 565
query63 247 208 202 202
query64 2409 814 644 644
query65
query66 1718 512 353 353
query67 29825 30024 29136 29136
query68
query69 468 339 303 303
query70 1003 987 938 938
query71 309 274 269 269
query72 2967 2738 2385 2385
query73 849 748 439 439
query74 5040 4929 4721 4721
query75 2631 2597 2251 2251
query76 2262 1138 761 761
query77 392 396 328 328
query78 12137 12141 11607 11607
query79 1462 1052 721 721
query80 634 549 460 460
query81 456 276 244 244
query82 1361 158 118 118
query83 358 274 249 249
query84 253 136 112 112
query85 887 534 456 456
query86 383 310 327 310
query87 3401 3403 3210 3210
query88 3514 2613 2592 2592
query89 438 383 339 339
query90 1849 176 182 176
query91 171 171 142 142
query92 75 80 74 74
query93 1500 1454 901 901
query94 535 344 297 297
query95 682 464 343 343
query96 1079 773 316 316
query97 2709 2702 2591 2591
query98 232 231 231 231
query99 1127 1098 984 984
Total cold run time: 253394 ms
Total hot run time: 168380 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3254045125
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java:
##########
@@ -98,21 +101,47 @@ public void collect(Expression expression) {
expression.accept(this, new CollectorContext(statementContext, bottomPredicate));
}
+ public void collectWholeVariantExpression(Expression expression) {
+ CollectorContext context = new CollectorContext(statementContext, bottomPredicate);
+ context.setCollectVariantRoot(true);
+ expression.accept(this, context);
+ }
+
private Void continueCollectAccessPath(Expression expr, CollectorContext context) {
return expr.accept(this, context);
}
+ private void recordVariantRootAccessPath(SlotReference slotReference, CollectorContext context) {
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId,
+ new CollectAccessPathResult(
+ ImmutableList.of(slotReference.getName()),
+ context.bottomFilter, ColumnAccessPathType.DATA));
+ }
+
@Override
public Void visit(Expression expr, CollectorContext context) {
for (Expression child : expr.children()) {
Review Comment:
This generic propagation still misses scalar functions that consume a whole VARIANT but return a non-VARIANT type. For example, `SELECT variant_type(v) FROM variant_tbl WHERE v['k'] IS NOT NULL` visits `variant_type(v)` with an empty context; because the function result is `STRING`, this branch sends `v` through a fresh empty context and records no root `[v]` demand. The filter can then leave only `[v, k]` at the scan, even though `variant_type(v)` needs the root VARIANT value/type and can produce wrong results when the root is a primitive/array/object or otherwise not represented by the selected subpath. Please make whole-VARIANT-consuming scalar functions such as `variant_type` force a root access, and add coverage combining that scalar output with a sibling nested VARIANT predicate. This is distinct from the existing whole-output/cast/dynamic-subscript/generator threads because the consumer here returns a non-VARIANT value, so the current whole-VARIANT-output handling never run
s.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469437098
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469405765
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470029801
# BE UT Coverage Report
Increment line coverage `75.58% (2655/3513)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/b88e809c8610f4487350cf3137ca8639091fde09_b88e809c8610f4487350cf3137ca8639091fde09/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/b88e809c8610f4487350cf3137ca8639091fde09_b88e809c8610f4487350cf3137ca8639091fde09/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.68% (20829/38804) |
| Line Coverage | 37.39% (197573/528361) |
| Region Coverage | 33.77% (154744/458212) |
| Branch Coverage | 34.87% (67839/194535) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4470043023
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469357514
<details>
<summary>TPC-DS: <b>Total hot run time: 169133 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit ada105e913ae461cd4e108f3ab58c331fbb2e237, data reload: false
query5 4309 632 517 517
query6 348 235 206 206
query7 4273 555 294 294
query8 334 246 224 224
query9 8818 3983 3976 3976
query10 459 340 294 294
query11 5823 2365 2243 2243
query12 184 128 128 128
query13 1280 592 419 419
query14 5980 5334 5003 5003
query14_1 4326 4319 4319 4319
query15 206 206 182 182
query16 1026 441 346 346
query17 1134 718 576 576
query18 2458 497 340 340
query19 212 201 162 162
query20 136 133 133 133
query21 220 138 116 116
query22 13578 13463 13446 13446
query23 17123 16372 15995 15995
query23_1 16183 16054 16252 16054
query24 7430 1773 1302 1302
query24_1 1311 1336 1309 1309
query25 586 504 449 449
query26 1303 330 179 179
query27 2681 550 338 338
query28 4505 1955 1924 1924
query29 1017 646 528 528
query30 301 242 200 200
query31 1113 1078 938 938
query32 94 86 76 76
query33 558 364 302 302
query34 1171 1149 629 629
query35 758 783 681 681
query36 1343 1344 1163 1163
query37 153 106 104 104
query38 3209 3148 3069 3069
query39 935 917 885 885
query39_1 889 885 868 868
query40 235 153 130 130
query41 74 69 68 68
query42 116 113 108 108
query43 323 330 286 286
query44
query45 220 208 198 198
query46 1072 1185 718 718
query47 2333 2346 2237 2237
query48 400 429 286 286
query49 646 537 384 384
query50 1061 369 256 256
query51 4304 4287 4233 4233
query52 107 104 93 93
query53 260 274 210 210
query54 308 274 267 267
query55 91 90 83 83
query56 301 295 314 295
query57 1418 1381 1319 1319
query58 296 276 275 275
query59 1544 1629 1459 1459
query60 319 319 313 313
query61 156 149 154 149
query62 681 633 565 565
query63 249 198 200 198
query64 2428 804 640 640
query65
query66 1724 478 361 361
query67 30163 30064 29951 29951
query68
query69 468 335 308 308
query70 946 989 908 908
query71 312 269 269 269
query72 3043 2693 2460 2460
query73 817 762 395 395
query74 5063 4866 4780 4780
query75 2711 2603 2247 2247
query76 2299 1109 742 742
query77 399 410 318 318
query78 12083 12155 11509 11509
query79 1433 1091 721 721
query80 638 540 460 460
query81 451 287 248 248
query82 1352 161 130 130
query83 363 279 246 246
query84 263 140 114 114
query85 890 534 452 452
query86 384 350 333 333
query87 3379 3366 3242 3242
query88 3550 2673 2624 2624
query89 445 385 338 338
query90 1983 181 175 175
query91 180 166 147 147
query92 74 78 76 76
query93 1443 1470 871 871
query94 548 357 300 300
query95 677 407 429 407
query96 1012 799 333 333
query97 2692 2734 2537 2537
query98 238 231 229 229
query99 1106 1127 983 983
Total cold run time: 252779 ms
Total hot run time: 169133 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4469344594
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468992253
run nonConcurrent
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468999567
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "raghav-reglobe (via GitHub)" <gi...@apache.org>.
raghav-reglobe commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-5118430543
One more from the same deployment, this time in the v1 read path itself: unshredded VARIANT data containing an **array of objects** (ubiquitous in MongoDB-shaped documents) — or a decimal on a JSONB-folding path — fails the whole file read with `[NOT_IMPLEMENTED_ERROR] Not implemeted`. The reader decodes these correctly; the failure is downstream in `FieldVisitorToJsonb`'s `VariantMap`/`Decimal*` stub arms when `ColumnVariant::Subcolumn::insert` folds to the JSONB least common type. Affects files written by any engine, so for us it means effectively every unshredded file of such collections.
PR with the fix (BE-UT-gated, 18/18 `ConvertFieldToTypeTest`), targeting this PR's branch: eldenmoon/incubator-doris#8 — fold it in directly if that's easier.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4458045379
<details>
<summary>TPC-H: <b>Total hot run time: 31056 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit e42bb8169a7ca02afa39b1f28690edfdbe5bfa75, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 18097 3976 3995 3976
q2 q3 10809 1435 836 836
q4 4694 470 346 346
q5 7576 2270 2109 2109
q6 226 172 138 138
q7 978 787 638 638
q8 9366 1752 1678 1678
q9 5158 4927 4878 4878
q10 6387 2076 1779 1779
q11 442 274 247 247
q12 639 425 291 291
q13 18166 3379 2813 2813
q14 263 257 230 230
q15 q16 816 782 706 706
q17 953 983 950 950
q18 7004 5728 5524 5524
q19 1310 1249 1023 1023
q20 505 413 266 266
q21 5842 2609 2319 2319
q22 451 358 309 309
Total cold run time: 99682 ms
Total hot run time: 31056 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4275 4136 4127 4127
q2 q3 4535 4887 4255 4255
q4 2109 2204 1371 1371
q5 4383 4268 4267 4267
q6 229 174 132 132
q7 2103 1986 1688 1688
q8 2528 2204 2172 2172
q9 7886 7915 7752 7752
q10 4539 4458 4087 4087
q11 584 416 374 374
q12 715 767 506 506
q13 3312 3617 2985 2985
q14 301 292 292 292
q15 q16 703 725 646 646
q17 1332 1320 1319 1319
q18 8064 7197 7432 7197
q19 1173 1161 1101 1101
q20 2218 2214 1919 1919
q21 5287 4607 4414 4414
q22 516 472 399 399
Total cold run time: 56792 ms
Total hot run time: 51003 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467050385
# BE UT Coverage Report
Increment line coverage `71.86% (1588/2210)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/67c53d3c346f8064df2eeeee8b5cd103c13cdd4f_67c53d3c346f8064df2eeeee8b5cd103c13cdd4f/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/67c53d3c346f8064df2eeeee8b5cd103c13cdd4f_67c53d3c346f8064df2eeeee8b5cd103c13cdd4f/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.59% (20748/38718) |
| Line Coverage | 37.30% (196687/527310) |
| Region Coverage | 33.68% (153956/457170) |
| Branch Coverage | 34.75% (67366/193870) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467049477
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467374978
# BE UT Coverage Report
Increment line coverage `75.72% (1987/2624)` :tada:
[Increment coverage report](http://coverage.selectdb-in.cc/coverage/89b6b024ee2a9aa5448ffd51797e4d5a5dd62147_89b6b024ee2a9aa5448ffd51797e4d5a5dd62147/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/89b6b024ee2a9aa5448ffd51797e4d5a5dd62147_89b6b024ee2a9aa5448ffd51797e4d5a5dd62147/report/index.html)
| Category | Coverage |
|-------------------|--------------------|
| Function Coverage | 53.63% (20778/38745) |
| Line Coverage | 37.33% (196923/527543) |
| Region Coverage | 33.73% (154311/457492) |
| Branch Coverage | 34.79% (67502/194049) |
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467388391
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467799286
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468238017
<details>
<summary>TPC-DS: <b>Total hot run time: 168970 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 81fcea94f01f769389c5f80754954b0057ad116c, data reload: false
query5 4325 661 497 497
query6 334 218 204 204
query7 4284 585 303 303
query8 324 230 218 218
query9 8781 4042 4067 4042
query10 456 327 297 297
query11 5808 2341 2251 2251
query12 180 126 122 122
query13 1281 622 446 446
query14 6076 5390 4990 4990
query14_1 4370 4363 4332 4332
query15 210 199 177 177
query16 1003 448 461 448
query17 1098 701 575 575
query18 2449 505 345 345
query19 208 194 159 159
query20 141 130 128 128
query21 223 143 118 118
query22 13561 13542 13315 13315
query23 17076 16364 15921 15921
query23_1 16019 16059 16075 16059
query24 7443 1803 1285 1285
query24_1 1312 1264 1305 1264
query25 550 502 444 444
query26 1318 322 174 174
query27 2703 569 361 361
query28 4482 1992 1924 1924
query29 1009 600 482 482
query30 302 232 193 193
query31 1108 1055 936 936
query32 91 73 70 70
query33 544 364 297 297
query34 1203 1137 640 640
query35 759 793 697 697
query36 1362 1349 1118 1118
query37 156 105 92 92
query38 3208 3152 3043 3043
query39 921 926 900 900
query39_1 850 862 879 862
query40 225 145 126 126
query41 65 64 62 62
query42 109 112 112 112
query43 328 332 290 290
query44
query45 217 200 194 194
query46 1052 1176 724 724
query47 2303 2326 2171 2171
query48 386 405 293 293
query49 641 495 386 386
query50 969 343 250 250
query51 4340 4232 4265 4232
query52 104 108 96 96
query53 259 288 202 202
query54 307 270 247 247
query55 90 88 83 83
query56 297 302 310 302
query57 1456 1366 1302 1302
query58 292 270 266 266
query59 1579 1605 1377 1377
query60 323 323 336 323
query61 153 152 153 152
query62 675 625 577 577
query63 245 204 201 201
query64 2399 778 631 631
query65
query66 1778 473 350 350
query67 30044 29891 29744 29744
query68
query69 468 344 296 296
query70 1040 1013 1005 1005
query71 319 277 257 257
query72 2981 2753 2598 2598
query73 856 774 408 408
query74 5062 4860 4776 4776
query75 2698 2617 2293 2293
query76 2313 1175 827 827
query77 419 426 333 333
query78 12124 12209 11640 11640
query79 1385 1058 754 754
query80 681 584 492 492
query81 462 283 246 246
query82 642 157 122 122
query83 368 287 260 260
query84 284 142 118 118
query85 960 608 542 542
query86 441 330 347 330
query87 3389 3407 3202 3202
query88 3572 2670 2676 2670
query89 434 382 336 336
query90 1899 176 182 176
query91 175 168 137 137
query92 75 79 72 72
query93 1445 1450 806 806
query94 561 354 320 320
query95 672 379 429 379
query96 1011 770 340 340
query97 2703 2668 2565 2565
query98 236 226 228 226
query99 1096 1087 966 966
Total cold run time: 251576 ms
Total hot run time: 168970 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468190224
<details>
<summary>TPC-H: <b>Total hot run time: 30909 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 81fcea94f01f769389c5f80754954b0057ad116c, data reload: false
------ Round 1 ----------------------------------
orders Doris NULL NULL 0 0 0 NULL 0 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 17790 3833 3813 3813
q2 q3 10771 1379 800 800
q4 4730 468 342 342
q5 7579 2217 2101 2101
q6 298 175 141 141
q7 971 789 624 624
q8 9411 1742 1532 1532
q9 7010 4867 4836 4836
q10 6428 2054 1762 1762
q11 453 277 241 241
q12 689 424 298 298
q13 18251 3729 2705 2705
q14 261 252 239 239
q15 q16 818 764 708 708
q17 955 951 917 917
q18 6735 5680 5428 5428
q19 1230 1321 1190 1190
q20 559 422 276 276
q21 5973 2711 2625 2625
q22 438 377 331 331
Total cold run time: 101350 ms
Total hot run time: 30909 ms
----- Round 2, with runtime_filter_mode=off -----
orders Doris NULL NULL 150000000 42 6422171781 NULL 22778155 NULL NULL 2023-12-26 18:27:23 2023-12-26 18:42:55 NULL utf-8 NULL NULL
============================================
q1 4611 4494 4568 4494
q2 q3 4850 5259 4574 4574
q4 2202 2152 1386 1386
q5 4777 4666 4686 4666
q6 226 177 133 133
q7 1829 1687 1543 1543
q8 2375 2130 1857 1857
q9 7179 7149 7141 7141
q10 4479 4357 3962 3962
q11 527 381 346 346
q12 695 719 507 507
q13 3031 3420 2770 2770
q14 278 280 250 250
q15 q16 673 693 611 611
q17 1244 1211 1224 1211
q18 7265 6791 6651 6651
q19 1080 1097 1120 1097
q20 2211 2205 1914 1914
q21 5235 4536 4418 4418
q22 510 454 429 429
Total cold run time: 55277 ms
Total hot run time: 49960 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468209444
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3253840980
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java:
##########
@@ -209,7 +213,7 @@ public Void visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, State
if (useWholeItem) {
// use the whole column
for (Expression child : function.children()) {
- exprCollector.collect(child);
+ exprCollector.collectWholeVariantExpression(child);
}
Review Comment:
The generator propagation added here still misses `ExplodeVariantArray`. That function is a separate generator class, so it falls through to the generic `exprCollector.collect(function)` branch below; with an access on the generated output such as `SELECT x['k'] FROM t LATERAL VIEW explode_variant_array(v) tmp AS x WHERE v[0]['p'] IS NOT NULL`, the output path on `x` is never propagated back to input `v`. The sibling predicate can then leave only a narrow `[v, 0, p]` access path at the scan, even though `explode_variant_array(v)` needs the full array/root to produce `x['k']`, causing missing/null results. This is distinct from the existing `LogicalGenerate` thread because that one covers regular `Explode`; please handle `ExplodeVariantArray` explicitly and add coverage combining generated-output access with a sibling VARIANT predicate.
##########
regression-test/suites/external_table_p0/tvf/test_local_tvf_iceberg_variant.groovy:
##########
@@ -0,0 +1,444 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.net.InetAddress
+import java.net.NetworkInterface
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+import org.apache.doris.regression.action.ProfileAction
+
+suite("test_local_tvf_iceberg_variant", "p0,external") {
+ List<List<Object>> backends = sql """ show backends """
+ assertTrue(backends.size() > 0)
+
+ def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/"
+ def beId = backends[0][0]
+ def outFilePath = "/"
+ def unshreddedData = "${dataFilePath}/iceberg_variant_unshredded.parquet"
+ def shreddedData = "${dataFilePath}/iceberg_variant_shredded.parquet"
+ def typedOnlyData = "${dataFilePath}/iceberg_variant_typed_only.parquet"
+ def temporalUnshreddedData = "${dataFilePath}/iceberg_variant_temporal_unshredded.parquet"
+ def temporalTypedData = "${dataFilePath}/iceberg_variant_temporal_typed.parquet"
+ def binaryUnshreddedData = "${dataFilePath}/iceberg_variant_binary_unshredded.parquet"
+ def binaryTypedData = "${dataFilePath}/iceberg_variant_binary_typed.parquet"
+
+ def localHosts = ["localhost", "127.0.0.1", InetAddress.localHost.hostAddress, InetAddress.localHost.hostName] as Set
+ NetworkInterface.networkInterfaces.each { networkInterface ->
+ networkInterface.inetAddresses.each { inetAddress ->
+ localHosts.add(inetAddress.hostAddress)
+ localHosts.add(inetAddress.hostName)
+ }
+ }
+
+ def dorisHome = new File(context.config.dataPath).parentFile.parentFile
+ def localBeHome = new File(dorisHome, "output/be")
+ def localJdbc = context.config.jdbcUrl.contains("127.0.0.1") || context.config.jdbcUrl.contains("localhost")
+ if (localJdbc && backends.size() == 1 && localHosts.contains(backends[0][1]) && localBeHome.exists()) {
+ outFilePath = ""
+ Files.copy(new File(unshreddedData).toPath(), new File(localBeHome, "iceberg_variant_unshredded.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ Files.copy(new File(shreddedData).toPath(), new File(localBeHome, "iceberg_variant_shredded.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ Files.copy(new File(typedOnlyData).toPath(), new File(localBeHome, "iceberg_variant_typed_only.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ Files.copy(new File(temporalUnshreddedData).toPath(), new File(localBeHome, "iceberg_variant_temporal_unshredded.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ Files.copy(new File(temporalTypedData).toPath(), new File(localBeHome, "iceberg_variant_temporal_typed.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ Files.copy(new File(binaryUnshreddedData).toPath(), new File(localBeHome, "iceberg_variant_binary_unshredded.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ Files.copy(new File(binaryTypedData).toPath(), new File(localBeHome, "iceberg_variant_binary_typed.parquet").toPath(),
+ StandardCopyOption.REPLACE_EXISTING)
+ } else {
+ for (List<Object> backend : backends) {
+ def beHost = backend[1]
+ scpFiles("root", beHost, unshreddedData, outFilePath, false)
+ scpFiles("root", beHost, shreddedData, outFilePath, false)
+ scpFiles("root", beHost, typedOnlyData, outFilePath, false)
+ scpFiles("root", beHost, temporalUnshreddedData, outFilePath, false)
+ scpFiles("root", beHost, temporalTypedData, outFilePath, false)
+ scpFiles("root", beHost, binaryUnshreddedData, outFilePath, false)
+ scpFiles("root", beHost, binaryTypedData, outFilePath, false)
+ }
+ }
+
+ def unshredded = outFilePath + "iceberg_variant_unshredded.parquet"
+ def shredded = outFilePath + "iceberg_variant_shredded.parquet"
+ def typedOnly = outFilePath + "iceberg_variant_typed_only.parquet"
+ def temporalUnshredded = outFilePath + "iceberg_variant_temporal_unshredded.parquet"
+ def temporalTyped = outFilePath + "iceberg_variant_temporal_typed.parquet"
+ def binaryUnshredded = outFilePath + "iceberg_variant_binary_unshredded.parquet"
+ def binaryTyped = outFilePath + "iceberg_variant_binary_typed.parquet"
+ def profileAction = new ProfileAction(context)
+ def getProfileByToken = { token ->
+ for (int i = 0; i < 60; ++i) {
+ List profileData = profileAction.getProfileList()
+ for (final def profileItem in profileData) {
+ if (profileItem["Sql Statement"].toString().contains(token)) {
+ def profileText = profileAction.getProfile(profileItem["Profile ID"].toString()).toString()
+ if (profileText.contains("ParquetReadColumnPaths")) {
+ return profileText
+ }
+ }
+ }
+ Thread.sleep(1000)
+ }
+ assertTrue(false)
+ }
+ def getParquetReadColumnPathSet = { profileText ->
+ def parquetReadColumnPaths = profileText.readLines().find { it.contains("ParquetReadColumnPaths") }
+ assertTrue(parquetReadColumnPaths != null)
+ logger.info("Iceberg variant shredding ${parquetReadColumnPaths}")
+ def separatorIndex = parquetReadColumnPaths.indexOf(":")
+ assertTrue(separatorIndex >= 0)
+ return parquetReadColumnPaths.substring(separatorIndex + 1)
+ .split(",")
+ .collect { it.trim() }
+ .findAll { !it.isEmpty() } as Set
+ }
+ def getProfileCounter = { profileText, counterName ->
+ def counterLine = profileText.readLines().find { it.contains(counterName) }
+ assertTrue(counterLine != null)
+ def matcher = counterLine =~ /${counterName}:\s*([0-9,]+)/
+ assertTrue(matcher.find())
+ return matcher.group(1).replace(",", "").toLong()
+ }
+
+ qt_desc_unshredded """
+ desc function local(
+ "file_path" = "${unshredded}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ """
+
+ qt_desc_typed_only """
+ desc function local(
+ "file_path" = "${typedOnly}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ """
+
+ order_qt_unshredded_complex """
+ select id,
+ cast(v['id'] as int) as variant_id,
+ cast(v['name'] as string) as name,
+ cast(v['metric'] as bigint) as metric,
+ cast(v['nested']['score'] as int) as score,
+ cast(v['nested']['flag'] as boolean) as flag,
+ cast(v['arr'] as array<int>)[1] as first_arr,
+ cast(v['arr'] as array<text>)[2] as second_arr
+ from local(
+ "file_path" = "${unshredded}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ order by id
+ """
+
+ order_qt_shredded_fields """
+ select id,
+ cast(v['metric'] as bigint) as metric,
+ cast(v['name'] as string) as name
+ from local(
+ "file_path" = "${shredded}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ order by id
+ """
+
+ order_qt_shredded_full_variant_with_scalar """
+ select id,
+ cast(v as string) like '%"name":"name-%' as has_name,
+ cast(v as string) like '%"metric":%' as has_metric
+ from local(
+ "file_path" = "${shredded}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ order by id
+ """
+
+ order_qt_typed_only_fields """
+ select id,
+ cast(v['metric'] as bigint) as metric,
+ cast(v['nested']['x'] as string) as nested_x,
+ cast(v['f'] as string) is null as non_finite_float_is_null,
+ cast(v['items'] as string) as items
+ from local(
+ "file_path" = "${typedOnly}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ order by id
+ """
+
+ order_qt_typed_only_missing_field """
+ select id,
+ cast(v['missing'] as string) is null as missing_is_null
+ from local(
+ "file_path" = "${typedOnly}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ order by id
+ """
+
+ order_qt_typed_only_nested_missing_field """
+ select id,
+ cast(v['nested']['missing'] as string) is null as missing_is_null
+ from local(
+ "file_path" = "${typedOnly}",
+ "backend_id" = "${beId}",
+ "format" = "parquet")
+ order by id
+ """
+
+ order_qt_temporal_parity """
+ select u.id,
+ cast(u.v['d'] as bigint) as unshredded_date,
+ cast(t.v['d'] as bigint) as typed_date,
+ cast(u.v['t'] as bigint) as unshredded_time,
+ cast(t.v['t'] as bigint) as typed_time,
+ cast(u.v['ts'] as bigint) as unshredded_ts,
+ cast(t.v['ts'] as bigint) as typed_ts,
+ cast(u.v['d'] as bigint) = cast(t.v['d'] as bigint) as same_date,
+ cast(u.v['t'] as bigint) = cast(t.v['t'] as bigint) as same_time,
+ cast(u.v['ts'] as bigint) = cast(t.v['ts'] as bigint) as same_ts
+ from local(
+ "file_path" = "${temporalUnshredded}",
+ "backend_id" = "${beId}",
+ "format" = "parquet") u
+ join local(
+ "file_path" = "${temporalTyped}",
+ "backend_id" = "${beId}",
+ "format" = "parquet") t
+ on u.id = t.id
+ order by u.id
+ """
+
+ def binaryUnshreddedRows = sql """
+ select id, hex(cast(v['b'] as varbinary))
+ from local(
+ "file_path" = "${binaryUnshredded}",
+ "backend_id" = "${beId}",
+ "format" = "parquet",
+ "enable_mapping_varbinary" = "true")
+ order by id
+ """
+ assertEquals(2, binaryUnshreddedRows.size())
+ assertEquals("1", binaryUnshreddedRows[0][0].toString())
+ assertEquals("FF0041", binaryUnshreddedRows[0][1].toString())
+ assertEquals("2", binaryUnshreddedRows[1][0].toString())
+ assertEquals("C328", binaryUnshreddedRows[1][1].toString())
+
+ def binaryTypedRows = sql """
+ select id, hex(cast(v['b'] as varbinary))
+ from local(
+ "file_path" = "${binaryTyped}",
+ "backend_id" = "${beId}",
+ "format" = "parquet",
+ "enable_mapping_varbinary" = "true")
+ order by id
+ """
+ assertEquals(2, binaryTypedRows.size())
+ assertEquals("1", binaryTypedRows[0][0].toString())
+ assertEquals("FF0041", binaryTypedRows[0][1].toString())
+ assertEquals("2", binaryTypedRows[1][0].toString())
+ assertEquals("C328", binaryTypedRows[1][1].toString())
+
+ sql """ set enable_profile = true """
+ sql """ set profile_level = 2 """
Review Comment:
This suite enables profiling for many subsequent assertions but only disables it at the very end. If any assertion or query between here and the final `set enable_profile = false` fails, the regression session continues with profiling enabled, which can pollute later profile-based checks and add avoidable overhead. Please wrap the profile section in `try/finally` (or otherwise restore both profile settings reliably) so failures do not leak session state.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4468734605
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4434038908
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3229682477
##########
be/src/format/table/hive/hive_parquet_nested_column_utils.cpp:
##########
@@ -18,20 +18,176 @@
#include "format/table/hive/hive_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_name(const FieldSchema& field_schema, std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+bool extract_nested_column_ids_by_name(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+bool extract_variant_nested_column_ids(const FieldSchema& variant_field,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_name(variant_field, "typed_value");
+ bool has_child_columns = false;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ has_child_columns = true;
+ continue;
+ }
+
+ add_variant_metadata(variant_field, column_ids);
+ bool found_typed_path = false;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ found_typed_path = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ found_typed_path = extract_nested_column_ids_by_name(*typed_child, child_paths,
Review Comment:
Fixed in latest push. VARIANT user keys now use exact matching while structural Parquet fields still use structural lookup. The pruning helper now treats unannotated value/typed_value shredded field groups as variant-like, so v['metric']['x'] selects v.typed_value.metric.value instead of top-level v.value. Added profile assertions for the deeper residual path.
##########
be/src/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp:
##########
@@ -18,21 +18,177 @@
#include "format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <iostream>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_name(const FieldSchema& field_schema, std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
+ return &child;
+ }
+ }
+ return nullptr;
+}
+
+void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ if (const auto* metadata = find_child_by_name(variant_field, "metadata")) {
+ add_column_id_range(*metadata, column_ids);
+ }
+}
+
+void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) {
+ add_variant_metadata(variant_field, column_ids);
+ if (const auto* value = find_child_by_name(variant_field, "value")) {
+ add_column_id_range(*value, column_ids);
+ }
+}
+
+bool extract_nested_column_ids_by_name(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+bool extract_variant_nested_column_ids(const FieldSchema& variant_field,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ const auto* typed_value = find_child_by_name(variant_field, "typed_value");
+ bool has_child_columns = false;
+
+ for (const auto& path : paths) {
+ if (path.empty()) {
+ add_column_id_range(variant_field, column_ids);
+ has_child_columns = true;
+ continue;
+ }
+
+ add_variant_metadata(variant_field, column_ids);
+ bool found_typed_path = false;
+ if (typed_value != nullptr) {
+ if (const auto* typed_child = find_child_by_name(*typed_value, path[0])) {
+ if (path.size() == 1) {
+ add_column_id_range(*typed_child, column_ids);
+ found_typed_path = true;
+ } else {
+ std::vector<std::vector<std::string>> child_paths {
+ std::vector<std::string>(path.begin() + 1, path.end())};
+ found_typed_path = extract_nested_column_ids_by_name(*typed_child, child_paths,
Review Comment:
Fixed in latest push. VARIANT user keys now use exact matching while structural Parquet fields still use structural lookup. The pruning helper now treats unannotated value/typed_value shredded field groups as variant-like, so v['metric']['x'] selects v.typed_value.metric.value instead of top-level v.value. Added profile assertions for the deeper residual path.
##########
be/src/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp:
##########
@@ -18,21 +18,177 @@
#include "format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <iostream>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_name(const FieldSchema& field_schema, std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
Review Comment:
Fixed in latest push. VARIANT path keys now use exact matching, so v['Name'] does not match shredded field name and falls back to v.value. Added a regression profile assertion that requires v.metadata and v.value while rejecting v.typed_value.name.
##########
be/src/format/table/hive/hive_parquet_nested_column_utils.cpp:
##########
@@ -18,20 +18,176 @@
#include "format/table/hive/hive_parquet_nested_column_utils.h"
#include <algorithm>
+#include <cctype>
#include <memory>
#include <set>
#include <string>
+#include <string_view>
#include <unordered_map>
#include <vector>
#include "format/parquet/schema_desc.h"
#include "format/table/table_schema_change_helper.h"
namespace doris {
+namespace {
+
+void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) {
+ const uint64_t start_id = field_schema.get_column_id();
+ const uint64_t max_column_id = field_schema.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+}
+
+const FieldSchema* find_child_by_name(const FieldSchema& field_schema, std::string_view name) {
+ std::string lower_name(name);
+ std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ for (const auto& child : field_schema.children) {
+ if (child.name == name || child.lower_case_name == lower_name) {
Review Comment:
Fixed in latest push. VARIANT path keys now use exact matching, so v['Name'] does not match shredded field name and falls back to v.value. Added a regression profile assertion that requires v.metadata and v.value while rejecting v.typed_value.name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "hello-stephen (via GitHub)" <gi...@apache.org>.
hello-stephen commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4436562770
<details>
<summary>TPC-DS: <b>Total hot run time: 172054 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit e9a298bb259994568bce23f301f467697fe37055, data reload: false
query5 4301 645 526 526
query6 323 216 201 201
query7 4234 529 326 326
query8 335 232 217 217
query9 8816 4107 4067 4067
query10 451 347 294 294
query11 5818 2406 2203 2203
query12 181 133 130 130
query13 1322 627 400 400
query14 6859 5382 5070 5070
query14_1 4396 4401 4388 4388
query15 214 203 186 186
query16 1047 464 468 464
query17 1187 778 647 647
query18 2753 495 364 364
query19 254 204 159 159
query20 137 134 130 130
query21 217 140 116 116
query22 13661 13493 13346 13346
query23 17110 16570 16826 16570
query23_1 16311 16376 16341 16341
query24 7784 1828 1411 1411
query24_1 1383 1361 1419 1361
query25 597 560 463 463
query26 1385 320 193 193
query27 3098 608 358 358
query28 4458 2018 2011 2011
query29 1017 679 546 546
query30 302 237 197 197
query31 1137 1048 923 923
query32 80 75 69 69
query33 528 337 286 286
query34 1157 1120 631 631
query35 785 768 668 668
query36 1341 1300 1167 1167
query37 146 98 87 87
query38 3189 3177 3080 3080
query39 919 912 897 897
query39_1 864 882 861 861
query40 223 152 133 133
query41 62 61 60 60
query42 111 106 105 105
query43 336 335 289 289
query44
query45 207 199 193 193
query46 1097 1168 745 745
query47 2347 2314 2212 2212
query48 402 440 291 291
query49 646 530 435 435
query50 697 284 213 213
query51 4321 4240 4264 4240
query52 106 110 94 94
query53 269 285 207 207
query54 317 279 265 265
query55 99 91 86 86
query56 295 300 300 300
query57 1402 1416 1328 1328
query58 288 278 266 266
query59 1608 1625 1421 1421
query60 352 329 325 325
query61 162 159 163 159
query62 672 607 564 564
query63 255 200 202 200
query64 2366 846 656 656
query65
query66 1696 510 395 395
query67 29980 29966 29787 29787
query68
query69 444 332 308 308
query70 996 1013 942 942
query71 306 279 267 267
query72 3070 2908 2634 2634
query73 865 778 421 421
query74 5090 4930 4786 4786
query75 2812 2674 2357 2357
query76 2303 1146 769 769
query77 428 452 354 354
query78 12917 12987 12378 12378
query79 1483 1005 707 707
query80 989 579 505 505
query81 508 281 241 241
query82 1310 158 127 127
query83 354 275 242 242
query84 268 142 111 111
query85 938 515 443 443
query86 426 317 321 317
query87 3427 3370 3208 3208
query88 3562 2692 2676 2676
query89 446 382 351 351
query90 1848 194 186 186
query91 180 169 144 144
query92 83 75 73 73
query93 961 942 549 549
query94 606 334 309 309
query95 669 391 436 391
query96 1027 743 339 339
query97 2703 2702 2535 2535
query98 238 230 222 222
query99 1110 1122 988 988
Total cold run time: 255770 ms
Total hot run time: 172054 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4466729483
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](iceberg) Support reading Iceberg variant from Parquet [doris]
Posted by "eldenmoon (via GitHub)" <gi...@apache.org>.
eldenmoon commented on PR #63192:
URL: https://github.com/apache/doris/pull/63192#issuecomment-4467036437
/review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org