You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/31 08:40:40 UTC
(doris) branch branch-2.0 updated: [branch-2.0][fix](outfile) Fix parquet writer wrong result issue (#26027)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 79608f95d97 [branch-2.0][fix](outfile) Fix parquet writer wrong result issue (#26027)
79608f95d97 is described below
commit 79608f95d9715770b74cfb070faac6f19f457719
Author: Tiewei Fang <43...@users.noreply.github.com>
AuthorDate: Tue Oct 31 16:40:32 2023 +0800
[branch-2.0][fix](outfile) Fix parquet writer wrong result issue (#26027)
---
be/src/vec/runtime/vfile_result_writer.cpp | 2 +-
be/src/vec/runtime/vparquet_writer.cpp | 62 +++++++++++-----------
.../hive_read/parquet/test_hive_read_parquet.out | 16 +++---
.../parquet/test_hive_read_parquet.groovy | 2 +-
4 files changed, 42 insertions(+), 40 deletions(-)
diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp
index fd599b32cc5..4a215d7898a 100644
--- a/be/src/vec/runtime/vfile_result_writer.cpp
+++ b/be/src/vec/runtime/vfile_result_writer.cpp
@@ -468,12 +468,12 @@ Status VFileResultWriter::_create_new_file_if_exceed_size() {
Status VFileResultWriter::_close_file_writer(bool done) {
if (_vfile_writer) {
+ RETURN_IF_ERROR(_vfile_writer->close());
// we can not use _current_written_bytes to COUNTER_UPDATE(_written_data_bytes, _current_written_bytes)
// because it will call `write()` function of orc/parquet function in `_vfile_writer->close()`
// and the real written_len will increase
// and _current_written_bytes will less than _vfile_writer->written_len()
COUNTER_UPDATE(_written_data_bytes, _vfile_writer->written_len());
- RETURN_IF_ERROR(_vfile_writer->close());
_vfile_writer.reset(nullptr);
} else if (_file_writer_impl) {
RETURN_IF_ERROR(_file_writer_impl->close());
diff --git a/be/src/vec/runtime/vparquet_writer.cpp b/be/src/vec/runtime/vparquet_writer.cpp
index 0a6392319b7..8d986dac326 100644
--- a/be/src/vec/runtime/vparquet_writer.cpp
+++ b/be/src/vec/runtime/vparquet_writer.cpp
@@ -343,23 +343,28 @@ Status VParquetWriterWrapper::parse_schema() {
#define RETURN_WRONG_TYPE \
return Status::InvalidArgument("Invalid column type: {}", raw_column->get_name());
-#define DISPATCH_PARQUET_NUMERIC_WRITER(WRITER, COLUMN_TYPE, NATIVE_TYPE) \
- parquet::RowGroupWriter* rgWriter = get_rg_writer(); \
- parquet::WRITER* col_writer = static_cast<parquet::WRITER*>(rgWriter->column(i)); \
- if (null_map != nullptr) { \
- auto& null_data = assert_cast<const ColumnUInt8&>(*null_map).get_data(); \
- for (size_t row_id = 0; row_id < sz; row_id++) { \
- def_level[row_id] = null_data[row_id] == 0; \
- } \
- col_writer->WriteBatch(sz, def_level.data(), nullptr, \
- reinterpret_cast<const NATIVE_TYPE*>( \
- assert_cast<const COLUMN_TYPE&>(*col).get_data().data())); \
- } else if (const auto* not_nullable_column = check_and_get_column<const COLUMN_TYPE>(col)) { \
- col_writer->WriteBatch( \
- sz, nullable ? def_level.data() : nullptr, nullptr, \
- reinterpret_cast<const NATIVE_TYPE*>(not_nullable_column->get_data().data())); \
- } else { \
- RETURN_WRONG_TYPE \
+#define DISPATCH_PARQUET_NUMERIC_WRITER(WRITER, COLUMN_TYPE, NATIVE_TYPE) \
+ parquet::RowGroupWriter* rgWriter = get_rg_writer(); \
+ parquet::WRITER* col_writer = static_cast<parquet::WRITER*>(rgWriter->column(i)); \
+ if (null_map != nullptr) { \
+ auto& null_data = assert_cast<const ColumnUInt8&>(*null_map).get_data(); \
+ if (const auto* column = check_and_get_column<const COLUMN_TYPE>(col)) { \
+ for (size_t row_id = 0; row_id < sz; row_id++) { \
+ if (null_data[row_id] != 0) { \
+ single_def_level = 0; \
+ } \
+ col_writer->WriteBatch( \
+ 1, &single_def_level, nullptr, \
+ reinterpret_cast<const NATIVE_TYPE*>(&column->get_data()[row_id])); \
+ single_def_level = 1; \
+ } \
+ } \
+ } else if (const auto* not_nullable_column = check_and_get_column<const COLUMN_TYPE>(col)) { \
+ col_writer->WriteBatch( \
+ sz, nullable ? def_level.data() : nullptr, nullptr, \
+ reinterpret_cast<const NATIVE_TYPE*>(not_nullable_column->get_data().data())); \
+ } else { \
+ RETURN_WRONG_TYPE \
}
#define DISPATCH_PARQUET_COMPLEX_WRITER(COLUMN_TYPE) \
@@ -533,32 +538,31 @@ Status VParquetWriterWrapper::write(const Block& block) {
parquet::RowGroupWriter* rgWriter = get_rg_writer();
parquet::Int64Writer* col_writer =
static_cast<parquet::Int64Writer*>(rgWriter->column(i));
- uint64_t default_int64 = 0;
if (null_map != nullptr) {
auto& null_data = assert_cast<const ColumnUInt8&>(*null_map).get_data();
for (size_t row_id = 0; row_id < sz; row_id++) {
def_level[row_id] = null_data[row_id] == 0;
}
int64_t tmp_data[sz];
+ int idx = 0;
for (size_t row_id = 0; row_id < sz; row_id++) {
- if (null_data[row_id] != 0) {
- tmp_data[row_id] = default_int64;
- } else {
+ if (null_data[row_id] == 0) {
VecDateTimeValue datetime_value = binary_cast<Int64, VecDateTimeValue>(
assert_cast<const ColumnVector<Int64>&>(*col)
.get_data()[row_id]);
- if (!datetime_value.unix_timestamp(&tmp_data[row_id],
+ if (!datetime_value.unix_timestamp(&tmp_data[idx],
TimezoneUtils::default_time_zone)) {
return Status::InternalError("get unix timestamp error.");
}
// -2177481943 represent '1900-12-31 23:54:17'
// but -2177481944 represent '1900-12-31 23:59:59'
// so for timestamp <= -2177481944, we subtract 343 (5min 43s)
- if (tmp_data[row_id] < timestamp_threshold) {
- tmp_data[row_id] -= timestamp_diff;
+ if (tmp_data[idx] < timestamp_threshold) {
+ tmp_data[idx] -= timestamp_diff;
}
// convert seconds to MILLIS seconds
- tmp_data[row_id] *= 1000;
+ tmp_data[idx] *= 1000;
+ ++idx;
}
}
col_writer->WriteBatch(sz, def_level.data(), nullptr,
@@ -594,7 +598,6 @@ Status VParquetWriterWrapper::write(const Block& block) {
parquet::RowGroupWriter* rgWriter = get_rg_writer();
parquet::Int64Writer* col_writer =
static_cast<parquet::Int64Writer*>(rgWriter->column(i));
- uint64_t default_int64 = 0;
if (null_map != nullptr) {
auto& null_data = assert_cast<const ColumnUInt8&>(*null_map).get_data();
for (size_t row_id = 0; row_id < sz; row_id++) {
@@ -607,15 +610,14 @@ Status VParquetWriterWrapper::write(const Block& block) {
}
int32_t days_from_epoch = epoch_date.daynr();
int32_t tmp_data[sz];
+ int idx = 0;
for (size_t row_id = 0; row_id < sz; row_id++) {
- if (null_data[row_id] != 0) {
- tmp_data[row_id] = default_int64;
- } else {
+ if (null_data[row_id] == 0) {
int32_t days = binary_cast<Int64, VecDateTimeValue>(
assert_cast<const ColumnVector<Int64>&>(*col)
.get_data()[row_id])
.daynr();
- tmp_data[row_id] = days - days_from_epoch;
+ tmp_data[idx++] = days - days_from_epoch;
}
}
col_writer->WriteBatch(sz, def_level.data(), nullptr,
diff --git a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out
index b2a579e9698..4a6bc2065c7 100644
--- a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out
+++ b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out
@@ -36,14 +36,14 @@
4 0000-01-01 0000-01-01 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000
-- !select_tvf2 --
-1 2023-04-20 2023-04-20 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
-2 9999-12-31 9999-12-31 9999-12-31 23:59:59 9999-12-31 23:59:59 2023-04-20 00:00:00.120000 2023-04-20 00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
-3 2023-04-21 2023-04-21 2023-04-20 12:34:56 2023-04-20 00:00:00 2023-04-20 00:00:00.123000 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678
-4 0000-01-01 0000-01-01 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000
+1 2023-04-20 2023-04-20 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00.000 2023-04-20 00:00:00.000000 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
+2 9999-12-31 9999-12-31 9999-12-31 23:59:59 9999-12-31 23:59:59 2023-04-20 00:00:00.120 2023-04-20 00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
+3 2023-04-21 2023-04-21 2023-04-20 12:34:56 2023-04-20 00:00:00 2023-04-20 00:00:00.123 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678
+4 0000-01-01 0000-01-01 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00.000 2023-04-20 00:00:00.000000 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000
-- !hive_docker_02 --
-1 2023-04-20 2023-04-20 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
-2 9999-12-31 9999-12-31 9999-12-31 23:59:59 9999-12-31 23:59:59 2023-04-20 00:00:00.120000 2023-04-20 00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
-3 2023-04-21 2023-04-21 2023-04-20 12:34:56 2023-04-20 00:00:00 2023-04-20 00:00:00.123000 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235E38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678
-4 0000-01-01 0000-01-01 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000
+1 2023-04-20 2023-04-20 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00.000 2023-04-20 00:00:00.000000 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
+2 9999-12-31 9999-12-31 9999-12-31 23:59:59 9999-12-31 23:59:59 2023-04-20 00:00:00.120 2023-04-20 00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
+3 2023-04-21 2023-04-21 2023-04-20 12:34:56 2023-04-20 00:00:00 2023-04-20 00:00:00.123 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235E38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678
+4 0000-01-01 0000-01-01 2023-04-20 00:00:00 2023-04-20 00:00:00 2023-04-20 00:00:00.000 2023-04-20 00:00:00.000000 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000
diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy
index 13a7b8e960e..428b97187dd 100644
--- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy
+++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy
@@ -199,7 +199,7 @@ suite("test_hive_read_parquet", "external,hive,external_docker") {
largeint_col STRING,
`float_col` float,
`double_col` double,
- `char_col` char(5),
+ `char_col` STRING,
`decimal_col` decimal ,
`decimalv3_col` decimal ,
`decimalv3_col2` decimal(1,0) ,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org