You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/23 07:39:19 UTC
[doris] branch dev-1.0.1 updated (cc7da636db -> a59b7ab13d)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from cc7da636db [fix](partition-cache) fix result may not write when enable partition cache (#10319)
new 363ea2b267 [fix][ldap] fix ldap password null pointer exception. (#10284)
new 9c8510b99a [fix][vectorized] Fix bug the of window function result not match plan nullable (#10340)
new e00e00c607 [fix] do not read seq column when reading a compacted rowset (#10344)
new a59b7ab13d [fixbug]opt nullable (#10346)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/olap_scanner.cpp | 37 +++++++++++++---------
be/src/exec/olap_scanner.h | 10 +++---
be/src/olap/rowset/segment_v2/column_reader.cpp | 10 ++++--
.../apache/doris/analysis/AggregateInfoBase.java | 2 +-
.../org/apache/doris/analysis/AnalyticInfo.java | 18 +++++++++++
.../java/org/apache/doris/ldap/LdapClient.java | 4 +--
.../java/org/apache/doris/persist/LdapInfo.java | 4 +++
7 files changed, 59 insertions(+), 26 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/04: [fix] do not read seq column when reading a compacted rowset (#10344)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e00e00c60748a794026fa19260fbcc10a5885373
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Thu Jun 23 08:44:43 2022 +0800
[fix] do not read seq column when reading a compacted rowset (#10344)
SEQ_COL is used on tables with unique key to order data in one transaction(rowset),
when there is only one rowset and the rowset is compacted, rows in the rowset is sorted
and rows with same keys are resolved by compaction, so a scanner sets direct_mode to
optimize read iterator to avoid sorting and aggregating, and iterators does not need SEQ_COL.
However, init_return_columns adds SEQ_COL to return_columns, which is passed to SegmentIterator.
Then segment Iterator would be called via get_next with a block without SEQ_COL, segment iterator
creates columns included in return_columns but not in the block. SEQ_COL is nullable, segment Iterator
does not handle it, so a core dump happen.
Actually, in the above case, segment iterator does not need to read SEQ_COL.
When SEQ_COL is really needed, iterators creates SEQ_COL column in block,
so segment Iterator does not need do create SEQ_COL at all.
---
be/src/exec/olap_scanner.cpp | 37 ++++++++++++++++++++++---------------
be/src/exec/olap_scanner.h | 10 +++++-----
2 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 4e2003ae0b..11a14a55da 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -138,7 +138,25 @@ Status OlapScanner::_init_tablet_reader_params(
const std::vector<OlapScanRange*>& key_ranges, const std::vector<TCondition>& filters,
const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>&
bloom_filters) {
- RETURN_IF_ERROR(_init_return_columns());
+ // if the table with rowset [0-x] or [0-1] [2-y], and [0-1] is empty
+ bool single_version =
+ (_tablet_reader_params.rs_readers.size() == 1 &&
+ _tablet_reader_params.rs_readers[0]->rowset()->start_version() == 0 &&
+ !_tablet_reader_params.rs_readers[0]
+ ->rowset()
+ ->rowset_meta()
+ ->is_segments_overlapping()) ||
+ (_tablet_reader_params.rs_readers.size() == 2 &&
+ _tablet_reader_params.rs_readers[0]->rowset()->rowset_meta()->num_rows() == 0 &&
+ _tablet_reader_params.rs_readers[1]->rowset()->start_version() == 2 &&
+ !_tablet_reader_params.rs_readers[1]
+ ->rowset()
+ ->rowset_meta()
+ ->is_segments_overlapping());
+
+ _tablet_reader_params.direct_mode = single_version || _aggregation;
+
+ RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
_tablet_reader_params.tablet = _tablet;
_tablet_reader_params.reader_type = READER_QUERY;
@@ -169,22 +187,11 @@ Status OlapScanner::_init_tablet_reader_params(
// TODO(zc)
_tablet_reader_params.profile = _parent->runtime_profile();
_tablet_reader_params.runtime_state = _runtime_state;
- // if the table with rowset [0-x] or [0-1] [2-y], and [0-1] is empty
- bool single_version =
- (_tablet_reader_params.rs_readers.size() == 1 &&
- _tablet_reader_params.rs_readers[0]->rowset()->start_version() == 0 &&
- !_tablet_reader_params.rs_readers[0]->rowset()->rowset_meta()->is_segments_overlapping()) ||
- (_tablet_reader_params.rs_readers.size() == 2 &&
- _tablet_reader_params.rs_readers[0]->rowset()->rowset_meta()->num_rows() == 0 &&
- _tablet_reader_params.rs_readers[1]->rowset()->start_version() == 2 &&
- !_tablet_reader_params.rs_readers[1]->rowset()->rowset_meta()->is_segments_overlapping());
-
_tablet_reader_params.origin_return_columns = &_return_columns;
_tablet_reader_params.tablet_columns_convert_to_null_set = &_tablet_columns_convert_to_null_set;
- if (_aggregation || single_version) {
+ if (_tablet_reader_params.direct_mode) {
_tablet_reader_params.return_columns = _return_columns;
- _tablet_reader_params.direct_mode = true;
} else {
// we need to fetch all key columns to do the right aggregation on storage engine side.
for (size_t i = 0; i < _tablet->num_key_columns(); ++i) {
@@ -219,7 +226,7 @@ Status OlapScanner::_init_tablet_reader_params(
return Status::OK();
}
-Status OlapScanner::_init_return_columns() {
+Status OlapScanner::_init_return_columns(bool need_seq_col) {
for (auto slot : _tuple_desc->slots()) {
if (!slot->is_materialized()) {
continue;
@@ -238,7 +245,7 @@ Status OlapScanner::_init_return_columns() {
}
// expand the sequence column
- if (_tablet->tablet_schema().has_sequence_col()) {
+ if (_tablet->tablet_schema().has_sequence_col() && need_seq_col) {
bool has_replace_col = false;
for (auto col : _return_columns) {
if (_tablet->tablet_schema().column(col).aggregation() ==
diff --git a/be/src/exec/olap_scanner.h b/be/src/exec/olap_scanner.h
index 4a6ecbf06f..fa0d7e4578 100644
--- a/be/src/exec/olap_scanner.h
+++ b/be/src/exec/olap_scanner.h
@@ -93,11 +93,11 @@ public:
const std::vector<SlotDescriptor*>& get_query_slots() const { return _query_slots; }
protected:
- Status _init_tablet_reader_params(const std::vector<OlapScanRange*>& key_ranges,
- const std::vector<TCondition>& filters,
- const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>&
- bloom_filters);
- Status _init_return_columns();
+ Status _init_tablet_reader_params(
+ const std::vector<OlapScanRange*>& key_ranges, const std::vector<TCondition>& filters,
+ const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>&
+ bloom_filters);
+ Status _init_return_columns(bool need_seq_col);
void _convert_row_to_tuple(Tuple* tuple);
// Update profile that need to be reported in realtime.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/04: [fixbug]opt nullable (#10346)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a59b7ab13d8f5943621b4d62c0b96e7236ef00aa
Author: wangbo <wa...@apache.org>
AuthorDate: Thu Jun 23 12:37:43 2022 +0800
[fixbug]opt nullable (#10346)
Co-authored-by: Wang Bo <wa...@meituan.com>
---
be/src/olap/rowset/segment_v2/column_reader.cpp | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 929b04cf5e..6b7c9f4d83 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -610,9 +610,13 @@ Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr &d
DCHECK_EQ(this_run, num_rows);
} else {
*has_null = true;
- // todo(wb) add a DCHECK here to check whether type is column nullable
- for (size_t x = 0; x < this_run; x++) {
- dst->insert_data(nullptr, 0); // todo(wb) vectorized here
+ auto* null_col =
+ vectorized::check_and_get_column<vectorized::ColumnNullable>(dst);
+ if (null_col != nullptr) {
+ const_cast<vectorized::ColumnNullable*>(null_col)->insert_null_elements(
+ this_run);
+ } else {
+ return Status::InternalError("unexpected column type in column reader");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/04: [fix][ldap] fix ldap password null pointer exception. (#10284)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 363ea2b267b6a676a73ba692b1bf09b63f860c5c
Author: luozenglin <37...@users.noreply.github.com>
AuthorDate: Thu Jun 23 15:01:18 2022 +0800
[fix][ldap] fix ldap password null pointer exception. (#10284)
---
fe/fe-core/src/main/java/org/apache/doris/ldap/LdapClient.java | 4 ++--
fe/fe-core/src/main/java/org/apache/doris/persist/LdapInfo.java | 4 ++++
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapClient.java b/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapClient.java
index ede65287eb..8e7d893c61 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapClient.java
@@ -102,13 +102,13 @@ public class LdapClient {
}
public boolean checkUpdate(String ldapPassword) {
- return !this.ldapPassword.equals(ldapPassword);
+ return this.ldapPassword == null || !this.ldapPassword.equals(ldapPassword);
}
}
private static void init() {
LdapInfo ldapInfo = Catalog.getCurrentCatalog().getAuth().getLdapInfo();
- if (ldapInfo == null) {
+ if (ldapInfo == null || !ldapInfo.isValid()) {
LOG.error("info is null, maybe no ldap admin password is set.");
ErrorReport.report(ErrorCode.ERROR_LDAP_CONFIGURATION_ERR);
throw new RuntimeException("ldapTemplate is not initialized");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/LdapInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/LdapInfo.java
index f985ef91c4..7b447ad962 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/LdapInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/LdapInfo.java
@@ -46,6 +46,10 @@ public class LdapInfo implements Writable {
ldapPasswdEncrypted = SymmetricEncryption.encrypt(ldapPasswd, secretKey, iv);
}
+ public boolean isValid() {
+ return ldapPasswdEncrypted != null && secretKey != null && iv != null;
+ }
+
public String getLdapPasswdEncrypted() {
return ldapPasswdEncrypted;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/04: [fix][vectorized] Fix bug the of window function result not match plan nullable (#10340)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9c8510b99ae419e9ab5aa9a37de9edbcf7e62e7b
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Thu Jun 23 08:40:12 2022 +0800
[fix][vectorized] Fix bug the of window function result not match plan nullable (#10340)
Co-authored-by: lihaopeng <li...@baidu.com>
---
.../org/apache/doris/analysis/AggregateInfoBase.java | 2 +-
.../java/org/apache/doris/analysis/AnalyticInfo.java | 18 ++++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java
index ed016ef3f0..ad4dedd591 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java
@@ -115,7 +115,7 @@ public abstract class AggregateInfoBase {
* Also updates the appropriate substitution map, and creates and registers auxiliary
* equality predicates between the grouping slots and the grouping exprs.
*/
- private TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple) {
+ protected TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple) {
TupleDescriptor result =
analyzer.getDescTbl().createTupleDescriptor(
tupleDebugName() + (isOutputTuple ? "-out" : "-intermed"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticInfo.java
index d1bf918576..ab3765ad29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticInfo.java
@@ -105,6 +105,24 @@ public final class AnalyticInfo extends AggregateInfoBase {
return result;
}
+ /**
+ * Creates the intermediate and output tuple descriptors. If no agg expr has an
+ * intermediate type different from its output type, then only the output tuple
+ * descriptor is created and the intermediate tuple is set to the output tuple.
+ * TODO: Rethink we really need to use Analyticinfo be subclass of AggregateInfoBase,
+ * it seems only use the aggregateExprs
+ */
+ protected void createTupleDescs(Analyzer analyzer) {
+ // Create the intermediate tuple desc first, so that the tuple ids are increasing
+ // from bottom to top in the plan tree.
+ intermediateTupleDesc_ = createTupleDesc(analyzer, false);
+ if (requiresIntermediateTuple(aggregateExprs_, false)) {
+ outputTupleDesc_ = createTupleDesc(analyzer, true);
+ } else {
+ outputTupleDesc_ = intermediateTupleDesc_;
+ }
+ }
+
/**
* Returns the intersection of the partition exprs of all the
* analytic functions.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org