You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/31 15:38:58 UTC
[doris] branch branch-1.2-lts updated (5ba7b42b32 -> d9666a694b)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 5ba7b42b32 [thirdparty](libhdfs3) update libhdfs3 to 2.3.7 to fix kms seek bug (#16265)
new 296cd73c8c [improvement](reader) use union merge when rowset are noneoverlapping (#15749)
new fb7944cd93 [enhancement](FE)shut down fast throw JVM might do when throwing exception #16146
new a30881d770 [docker](iceberg) add iceberg docker compose and modify scripts (#16175)
new 73d6ffc47d [Bug](exec) enable warning on ignoring function return value for vctx (#16157)
new 691baff248 [Enhancement](profile) use queryId of originStatement as the queryId. (#15898)
new 24f2511a39 [Fix](profile) do not send export profile when enable_profile=false. (#15996)
new 749819c390 [Feature](profile) set sql and defaultDb fields in show-load-profile. (#15875)
new 4d65a0723a [Enhancement](profile) fill user field for profile. (#16212)
new bf5417e80a [Improvement](decimal) do not log fatal when precision is invalid (#16207)
new df5ff77587 [improvement](metrics) Metrics add all rowset nums and segment nums (#16208)
new 9be0b42569 [fix][FE] fix be coredump when children of FunctionCallExpr is folded (#16064)
new eadd33a609 [fix](vresultsink) BufferControlBlock may block all fragment handle threads (#16231)
new 5cf69f302b [fix](olap) Incorrect reserving size for PredicateColumn converted from ColumnDictionary (#16249)
new cb47682d09 [fix](planner) Pushdown constant predicate to all scan node in the lieteral view. #16217
new f1ae54992e [Bug](function) positive function coredump in decimal (#16230)
new 92df75c8c8 [Bug](function) fix now(int) use_default_implementation_for_nulls && fix dround signature (#16238)
new b13500f4db [Bug](date) Fix invalid date (#16205)
new f1539758e3 [enhencement](lock) print table lock owner when failed to try lock (#16186)
new 60f22608a4 [branch-1.2] format code after cherry-pick
new d9666a694b [improvement](agg-function) Increase the limit maximum number of agg function parameters (#15924)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.licenserc.yaml | 1 +
be/src/common/daemon.cpp | 6 +
be/src/olap/tablet.h | 11 +
be/src/olap/tablet_manager.cpp | 22 ++
be/src/olap/tablet_manager.h | 3 +
be/src/runtime/result_buffer_mgr.cpp | 10 +-
be/src/runtime/result_buffer_mgr.h | 2 +-
be/src/runtime/result_file_sink.cpp | 2 +-
be/src/runtime/result_sink.cpp | 4 +-
be/src/runtime/runtime_state.h | 1 +
be/src/util/doris_metrics.cpp | 6 +
be/src/util/doris_metrics.h | 3 +
.../aggregate_functions/aggregate_function_null.h | 4 +-
be/src/vec/columns/column_dictionary.h | 2 +-
be/src/vec/core/sort_cursor.h | 6 +-
be/src/vec/data_types/data_type_decimal.cpp | 6 +-
.../vec/exec/join/process_hash_table_probe_impl.h | 10 +-
be/src/vec/exec/vunion_node.cpp | 3 +-
be/src/vec/exprs/vexpr.cpp | 3 +
be/src/vec/exprs/vexpr_context.h | 15 +-
be/src/vec/exprs/vliteral.cpp | 28 ++-
.../function_date_or_datetime_computation.cpp | 2 +-
.../function_date_or_datetime_computation.h | 8 +-
be/src/vec/functions/function_unary_arithmetic.h | 7 +-
be/src/vec/functions/math.cpp | 1 +
be/src/vec/olap/block_reader.cpp | 36 ++-
be/src/vec/olap/block_reader.h | 4 +
be/src/vec/olap/vcollect_iterator.cpp | 61 ++++-
be/src/vec/olap/vcollect_iterator.h | 6 +-
be/src/vec/sink/vresult_file_sink.cpp | 2 +-
be/src/vec/sink/vresult_sink.cpp | 4 +-
be/test/vec/exprs/vexpr_test.cpp | 18 +-
bin/start_fe.sh | 4 +-
docker/thirdparties/custom_settings.env | 23 ++
.../elasticsearch/{es.yaml => es.yaml.tpl} | 0
...hadoop-hive.env.tpl => hadoop-hive.env.tpl.tpl} | 0
.../hive/{hive-2x.yaml => hive-2x.yaml.tpl} | 0
.../docker-compose/iceberg/entrypoint.sh.tpl | 34 +++
.../docker-compose/iceberg/iceberg.env | 24 ++
.../docker-compose/iceberg/iceberg.yaml.tpl | 96 ++++++++
.../docker-compose/iceberg/spark-defaults.conf.tpl | 11 +
.../mysql/{mysql-5.7.yaml => mysql-5.7.yaml.tpl} | 0
.../oracle/{oracle-11.yaml => oracle-11.yaml.tpl} | 2 +-
.../{postgresql-14.yaml => postgresql-14.yaml.tpl} | 0
.../{sqlserver.yaml => sqlserver.yaml.tpl} | 0
docker/thirdparties/run-thirdparties-docker.sh | 251 +++++++++++++++++++++
docker/thirdparties/start-thirdparties-docker.sh | 79 -------
docker/thirdparties/stop-thirdparties-docker.sh | 43 ----
.../developer-guide/regression-testing.md | 111 +++++++--
.../org/apache/doris/analysis/DateLiteral.java | 18 +-
.../apache/doris/analysis/ExpressionFunctions.java | 13 +-
.../doris/analysis/ShowQueryProfileStmt.java | 1 +
.../org/apache/doris/analysis/StringLiteral.java | 5 +
.../main/java/org/apache/doris/catalog/Table.java | 22 +-
.../apache/doris/common/util/ProfileManager.java | 21 +-
.../util/QueryableReentrantReadWriteLock.java | 41 ++++
.../main/java/org/apache/doris/load/ExportJob.java | 27 ++-
.../apache/doris/load/loadv2/BrokerLoadJob.java | 15 +-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 9 +
.../apache/doris/planner/SingleNodePlanner.java | 23 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 6 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 7 +
.../mvrewrite/CountDistinctToBitmapOrHLLRule.java | 2 +-
.../org/apache/doris/task/ExportExportingTask.java | 8 +-
.../org/apache/doris/qe/SessionVariablesTest.java | 131 +++++++++--
.../data/datatype_p0/date/test_invalid_date.out | 7 +
.../data/query_p0/literal_view/lietral_test.out | 3 +
.../test_aggregate_retention.out | 26 +++
.../test_count_distinct_with_case_function.out | 3 +
.../datatype_p0/date/test_invalid_date.groovy | 37 +++
.../query_p0/literal_view/lietral_test.groovy | 88 ++++++++
.../test_aggregate_retention.sql | 42 +++-
.../test_count_distinct_with_case_function.groovy | 58 +++++
73 files changed, 1318 insertions(+), 270 deletions(-)
create mode 100644 docker/thirdparties/custom_settings.env
rename docker/thirdparties/docker-compose/elasticsearch/{es.yaml => es.yaml.tpl} (100%)
rename docker/thirdparties/docker-compose/hive/{hadoop-hive.env.tpl => hadoop-hive.env.tpl.tpl} (100%)
rename docker/thirdparties/docker-compose/hive/{hive-2x.yaml => hive-2x.yaml.tpl} (100%)
create mode 100755 docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
create mode 100644 docker/thirdparties/docker-compose/iceberg/iceberg.env
create mode 100644 docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
create mode 100644 docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl
rename docker/thirdparties/docker-compose/mysql/{mysql-5.7.yaml => mysql-5.7.yaml.tpl} (100%)
rename docker/thirdparties/docker-compose/oracle/{oracle-11.yaml => oracle-11.yaml.tpl} (98%)
rename docker/thirdparties/docker-compose/postgresql/{postgresql-14.yaml => postgresql-14.yaml.tpl} (100%)
rename docker/thirdparties/docker-compose/sqlserver/{sqlserver.yaml => sqlserver.yaml.tpl} (100%)
create mode 100755 docker/thirdparties/run-thirdparties-docker.sh
delete mode 100755 docker/thirdparties/start-thirdparties-docker.sh
delete mode 100755 docker/thirdparties/stop-thirdparties-docker.sh
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java
create mode 100644 regression-test/data/datatype_p0/date/test_invalid_date.out
create mode 100644 regression-test/data/query_p0/literal_view/lietral_test.out
create mode 100644 regression-test/data/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.out
create mode 100644 regression-test/suites/datatype_p0/date/test_invalid_date.groovy
create mode 100644 regression-test/suites/query_p0/literal_view/lietral_test.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/20: [Enhancement](profile) fill user field for profile. (#16212)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4d65a0723a80a3fee5e46656eff7685239e1a73b
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Mon Jan 30 09:15:02 2023 +0800
[Enhancement](profile) fill user field for profile. (#16212)
Co-authored-by: wangxiangyu@360shuke.com <wa...@360shuke.com>
---
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java | 8 +++++++-
.../src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 3 ++-
.../src/main/java/org/apache/doris/task/ExportExportingTask.java | 2 +-
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 3b79ca70de..36bdaa993c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -120,6 +120,7 @@ public class ExportJob implements Writable {
private long id;
private String queryId;
private String label;
+ private String user;
private long dbId;
private long tableId;
private BrokerDesc brokerDesc;
@@ -192,6 +193,7 @@ public class ExportJob implements Writable {
this.columnSeparator = "\t";
this.lineDelimiter = "\n";
this.columns = "";
+ this.user = "";
}
public ExportJob(long jobId) {
@@ -209,7 +211,7 @@ public class ExportJob implements Writable {
this.properties = stmt.getProperties();
this.label = this.properties.get(ExportStmt.LABEL);
this.queryId = ConnectContext.get() != null ? DebugUtil.printId(ConnectContext.get().queryId()) : "N/A";
-
+ this.user = ConnectContext.get() != null ? ConnectContext.get().getQualifiedUser() : "N/A";
String path = stmt.getPath();
Preconditions.checkArgument(!Strings.isNullOrEmpty(path));
this.whereExpr = stmt.getWhereExpr();
@@ -751,6 +753,10 @@ public class ExportJob implements Writable {
return queryId;
}
+ public String getUser() {
+ return user;
+ }
+
public boolean getEnableProfile() {
return enableProfile;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index 31c07820cc..01bbd795f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -325,7 +325,8 @@ public class BrokerLoadJob extends BulkLoadJob {
summaryProfile.addInfoString(ProfileManager.QUERY_TYPE, "Load");
summaryProfile.addInfoString(ProfileManager.QUERY_STATE, "N/A");
- summaryProfile.addInfoString(ProfileManager.USER, "N/A");
+ summaryProfile.addInfoString(ProfileManager.USER,
+ getUserInfo() != null ? getUserInfo().getQualifiedUser() : "N/A");
summaryProfile.addInfoString(ProfileManager.DEFAULT_DB, getDefaultDb());
summaryProfile.addInfoString(ProfileManager.SQL_STATEMENT, this.getOriginStmt().originStmt);
summaryProfile.addInfoString(ProfileManager.IS_CACHED, "N/A");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
index 7d6a3b3c74..2a6bed2d9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
@@ -264,7 +264,7 @@ public class ExportExportingTask extends MasterTask {
summaryProfile.addInfoString(ProfileManager.QUERY_TYPE, "Export");
summaryProfile.addInfoString(ProfileManager.QUERY_STATE, job.getState().toString());
summaryProfile.addInfoString(ProfileManager.DORIS_VERSION, Version.DORIS_BUILD_VERSION);
- summaryProfile.addInfoString(ProfileManager.USER, "xxx");
+ summaryProfile.addInfoString(ProfileManager.USER, job.getUser());
summaryProfile.addInfoString(ProfileManager.DEFAULT_DB, String.valueOf(job.getDbId()));
summaryProfile.addInfoString(ProfileManager.SQL_STATEMENT, job.getSql());
profile.addChild(summaryProfile);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 15/20: [Bug](function) positive function coredump in decimal (#16230)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f1ae54992e3054c04d68d0193e45f05a8256a231
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Mon Jan 30 22:17:50 2023 +0800
[Bug](function) positive function coredump in decimal (#16230)
---
be/src/vec/functions/function_unary_arithmetic.h | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/functions/function_unary_arithmetic.h b/be/src/vec/functions/function_unary_arithmetic.h
index c92ceb54d1..8113f42179 100644
--- a/be/src/vec/functions/function_unary_arithmetic.h
+++ b/be/src/vec/functions/function_unary_arithmetic.h
@@ -54,14 +54,17 @@ template <typename>
struct AbsImpl;
template <typename>
struct NegativeImpl;
+template <typename>
+struct PositiveImpl;
/// Used to indicate undefined operation
struct InvalidType;
template <template <typename> class Op, typename Name, bool is_injective>
class FunctionUnaryArithmetic : public IFunction {
- static constexpr bool allow_decimal =
- std::is_same_v<Op<Int8>, NegativeImpl<Int8>> || std::is_same_v<Op<Int8>, AbsImpl<Int8>>;
+ static constexpr bool allow_decimal = std::is_same_v<Op<Int8>, NegativeImpl<Int8>> ||
+ std::is_same_v<Op<Int8>, AbsImpl<Int8>> ||
+ std::is_same_v<Op<Int8>, PositiveImpl<Int8>>;
template <typename F>
static bool cast_type(const IDataType* type, F&& f) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/20: [improvement](reader) use union merge when rowset are noneoverlapping (#15749)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 296cd73c8c429af26a187f38a0fc9f0f70be58c9
Author: yixiutt <10...@users.noreply.github.com>
AuthorDate: Mon Jan 16 21:53:18 2023 +0800
[improvement](reader) use union merge when rowset are noneoverlapping (#15749)
---
be/src/vec/olap/block_reader.cpp | 36 ++++++++++++++++++---
be/src/vec/olap/block_reader.h | 4 +++
be/src/vec/olap/vcollect_iterator.cpp | 61 ++++++++++++++++++++++++++++++++---
be/src/vec/olap/vcollect_iterator.h | 6 +++-
4 files changed, 97 insertions(+), 10 deletions(-)
diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index aa68e382b6..fa79e1bb9c 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -33,9 +33,33 @@ BlockReader::~BlockReader() {
}
}
+bool BlockReader::_rowsets_overlapping(const std::vector<RowsetReaderSharedPtr>& rs_readers) {
+ std::string cur_max_key;
+ for (const auto& rs_reader : rs_readers) {
+ // version 0-1 of every tablet is empty, just skip this rowset
+ if (rs_reader->rowset()->version().second == 1) {
+ continue;
+ }
+ if (rs_reader->rowset()->num_rows() == 0) {
+ continue;
+ }
+ if (rs_reader->rowset()->is_segments_overlapping()) {
+ return true;
+ }
+ std::string min_key;
+ bool has_min_key = rs_reader->rowset()->min_key(&min_key);
+ if (!has_min_key) {
+ return true;
+ }
+ if (min_key <= cur_max_key) {
+ return true;
+ }
+ CHECK(rs_reader->rowset()->max_key(&cur_max_key));
+ }
+ return false;
+}
Status BlockReader::_init_collect_iter(const ReaderParams& read_params,
std::vector<RowsetReaderSharedPtr>* valid_rs_readers) {
- _vcollect_iter.init(this, read_params.read_orderby_key, read_params.read_orderby_key_reverse);
std::vector<RowsetReaderSharedPtr> rs_readers;
auto res = _capture_rs_readers(read_params, &rs_readers);
if (!res.ok()) {
@@ -46,6 +70,10 @@ Status BlockReader::_init_collect_iter(const ReaderParams& read_params,
<< ", version:" << read_params.version;
return res;
}
+ // check if rowsets are noneoverlapping
+ _is_rowsets_overlapping = _rowsets_overlapping(rs_readers);
+ _vcollect_iter.init(this, _is_rowsets_overlapping, read_params.read_orderby_key,
+ read_params.read_orderby_key_reverse);
_reader_context.batch_size = _batch_size;
_reader_context.is_vec = true;
@@ -63,10 +91,8 @@ Status BlockReader::_init_collect_iter(const ReaderParams& read_params,
}
RETURN_IF_ERROR(_vcollect_iter.build_heap(*valid_rs_readers));
- if (_vcollect_iter.is_merge()) {
- auto status = _vcollect_iter.current_row(&_next_row);
- _eof = status.precise_code() == OLAP_ERR_DATA_EOF;
- }
+ auto status = _vcollect_iter.current_row(&_next_row);
+ _eof = status.is_end_of_file();
return Status::OK();
}
diff --git a/be/src/vec/olap/block_reader.h b/be/src/vec/olap/block_reader.h
index 0213783da0..83786a2e69 100644
--- a/be/src/vec/olap/block_reader.h
+++ b/be/src/vec/olap/block_reader.h
@@ -86,6 +86,8 @@ private:
bool _get_next_row_same();
+ bool _rowsets_overlapping(const std::vector<RowsetReaderSharedPtr>& rs_readers);
+
VCollectIterator _vcollect_iter;
IteratorRowRef _next_row {{}, -1, false};
@@ -115,6 +117,8 @@ private:
std::vector<RowLocation> _block_row_locations;
ColumnPtr _delete_filter_column;
+
+ bool _is_rowsets_overlapping = true;
};
} // namespace vectorized
diff --git a/be/src/vec/olap/vcollect_iterator.cpp b/be/src/vec/olap/vcollect_iterator.cpp
index b9160514ad..cc07e9be30 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -37,7 +37,8 @@ VCollectIterator::~VCollectIterator() {
}
}
-void VCollectIterator::init(TabletReader* reader, bool force_merge, bool is_reverse) {
+void VCollectIterator::init(TabletReader* reader, bool ori_data_overlapping, bool force_merge,
+ bool is_reverse) {
_reader = reader;
// when aggregate is enabled or key_type is DUP_KEYS, we don't merge
// multiple data to aggregate for better performance
@@ -47,8 +48,10 @@ void VCollectIterator::init(TabletReader* reader, bool force_merge, bool is_reve
_reader->_tablet->enable_unique_key_merge_on_write()))) {
_merge = false;
}
-
- if (force_merge) {
+ // When data is none overlapping, no need to build heap to traverse data
+ if (!ori_data_overlapping) {
+ _merge = false;
+ } else if (force_merge) {
_merge = true;
}
_is_reverse = is_reverse;
@@ -127,6 +130,22 @@ Status VCollectIterator::build_heap(std::vector<RowsetReaderSharedPtr>& rs_reade
new Level1Iterator(_children, _reader, _merge, _is_reverse, _skip_same));
}
} else {
+ bool have_multiple_child = false;
+ bool is_first_child = true;
+ for (auto iter = _children.begin(); iter != _children.end();) {
+ auto s = (*iter)->init_for_union(is_first_child, have_multiple_child);
+ if (!s.ok()) {
+ delete (*iter);
+ iter = _children.erase(iter);
+ if (!s.is<END_OF_FILE>()) {
+ return s;
+ }
+ } else {
+ have_multiple_child = true;
+ is_first_child = false;
+ ++iter;
+ }
+ }
_inner_iter.reset(new Level1Iterator(_children, _reader, _merge, _is_reverse, _skip_same));
}
RETURN_IF_NOT_EOF_AND_OK(_inner_iter->init());
@@ -212,6 +231,36 @@ Status VCollectIterator::Level0Iterator::init(bool get_data_by_ref) {
return st;
}
+// if is_first_child = true, return first row in block。Unique keys and agg keys will
+// read a line first and then start loop :
+// while (!eof) {
+// collect_iter->next(&_next_row);
+// }
+// so first child load first row and other child row_pos = -1
+Status VCollectIterator::Level0Iterator::init_for_union(bool is_first_child, bool get_data_by_ref) {
+ _get_data_by_ref = get_data_by_ref && _rs_reader->support_return_data_by_ref() &&
+ config::enable_storage_vectorization;
+ if (!_get_data_by_ref) {
+ _block = std::make_shared<Block>(_schema.create_block(
+ _reader->_return_columns, _reader->_tablet_columns_convert_to_null_set));
+ }
+ auto st = _refresh_current_row();
+ if (_get_data_by_ref && _block_view.size()) {
+ if (is_first_child) {
+ _ref = _block_view[0];
+ } else {
+ _ref = _block_view[-1];
+ }
+ } else {
+ if (is_first_child) {
+ _ref = {_block, 0, false};
+ } else {
+ _ref = {_block, -1, false};
+ }
+ }
+ return st;
+}
+
int64_t VCollectIterator::Level0Iterator::version() const {
return _rs_reader->version().second;
}
@@ -259,7 +308,7 @@ Status VCollectIterator::Level0Iterator::next(IteratorRowRef* ref) {
Status VCollectIterator::Level0Iterator::next(Block* block) {
CHECK(!_get_data_by_ref);
- if (_ref.row_pos == 0 && _ref.block != nullptr && UNLIKELY(_ref.block->rows() > 0)) {
+ if (_ref.row_pos <= 0 && _ref.block != nullptr && UNLIKELY(_ref.block->rows() > 0)) {
block->swap(*_ref.block);
_ref.reset();
return Status::OK();
@@ -306,6 +355,10 @@ VCollectIterator::Level1Iterator::Level1Iterator(
_skip_same(skip_same) {
_ref.reset();
_batch_size = reader->_batch_size;
+ // !_merge means that data are in order, so we just reverse children to return data in reverse
+ if (!_merge && _is_reverse) {
+ _children.reverse();
+ }
}
VCollectIterator::Level1Iterator::~Level1Iterator() {
diff --git a/be/src/vec/olap/vcollect_iterator.h b/be/src/vec/olap/vcollect_iterator.h
index b16f752bc0..d64e45cf91 100644
--- a/be/src/vec/olap/vcollect_iterator.h
+++ b/be/src/vec/olap/vcollect_iterator.h
@@ -39,7 +39,7 @@ public:
// Hold reader point to get reader params
~VCollectIterator();
- void init(TabletReader* reader, bool force_merge, bool is_reverse);
+ void init(TabletReader* reader, bool ori_data_overlapping, bool force_merge, bool is_reverse);
Status add_child(RowsetReaderSharedPtr rs_reader);
@@ -85,6 +85,9 @@ private:
_compare_columns(reader->_reader_context.read_orderby_key_columns) {};
virtual Status init(bool get_data_by_ref = false) = 0;
+ virtual Status init_for_union(bool is_first_child, bool get_data_by_ref = false) {
+ return Status::OK();
+ };
virtual int64_t version() const = 0;
@@ -146,6 +149,7 @@ private:
~Level0Iterator() override = default;
Status init(bool get_data_by_ref = false) override;
+ Status init_for_union(bool is_first_child, bool get_data_by_ref = false) override;
int64_t version() const override;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 17/20: [Bug](date) Fix invalid date (#16205)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b13500f4dbbdf276897956a1904c872745779d82
Author: Gabriel <ga...@gmail.com>
AuthorDate: Tue Jan 31 10:08:44 2023 +0800
[Bug](date) Fix invalid date (#16205)
Issue Number: close #15777
---
be/src/vec/exprs/vliteral.cpp | 28 ++++++++++------
.../org/apache/doris/analysis/DateLiteral.java | 18 +++++++++--
.../apache/doris/analysis/ExpressionFunctions.java | 13 +++++++-
.../org/apache/doris/analysis/StringLiteral.java | 5 +++
.../data/datatype_p0/date/test_invalid_date.out | 7 ++++
.../datatype_p0/date/test_invalid_date.groovy | 37 ++++++++++++++++++++++
6 files changed, 95 insertions(+), 13 deletions(-)
diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp
index 27324e5c0a..f78118ab0f 100644
--- a/be/src/vec/exprs/vliteral.cpp
+++ b/be/src/vec/exprs/vliteral.cpp
@@ -93,28 +93,36 @@ void VLiteral::init(const TExprNode& node) {
}
case TYPE_DATE: {
VecDateTimeValue value;
- value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size());
- value.cast_to_date();
- field = Int64(*reinterpret_cast<__int64_t*>(&value));
+ if (value.from_date_str(node.date_literal.value.c_str(),
+ node.date_literal.value.size())) {
+ value.cast_to_date();
+ field = Int64(*reinterpret_cast<__int64_t*>(&value));
+ }
break;
}
case TYPE_DATEV2: {
DateV2Value<DateV2ValueType> value;
- value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size());
- field = value.to_date_int_val();
+ if (value.from_date_str(node.date_literal.value.c_str(),
+ node.date_literal.value.size())) {
+ field = value.to_date_int_val();
+ }
break;
}
case TYPE_DATETIMEV2: {
DateV2Value<DateTimeV2ValueType> value;
- value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size());
- field = value.to_date_int_val();
+ if (value.from_date_str(node.date_literal.value.c_str(),
+ node.date_literal.value.size())) {
+ field = value.to_date_int_val();
+ }
break;
}
case TYPE_DATETIME: {
VecDateTimeValue value;
- value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size());
- value.to_datetime();
- field = Int64(*reinterpret_cast<__int64_t*>(&value));
+ if (value.from_date_str(node.date_literal.value.c_str(),
+ node.date_literal.value.size())) {
+ value.to_datetime();
+ field = Int64(*reinterpret_cast<__int64_t*>(&value));
+ }
break;
}
case TYPE_STRING:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
index a87811bcfe..d7615151f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -627,6 +627,13 @@ public class DateLiteral extends LiteralExpr {
}
msg.node_type = TExprNodeType.DATE_LITERAL;
msg.date_literal = new TDateLiteral(getStringValue());
+ try {
+ checkValueValid();
+ } catch (AnalysisException e) {
+ // If date value is invalid, set this to null
+ msg.node_type = TExprNodeType.NULL_LITERAL;
+ msg.setIsNullable(true);
+ }
}
@Override
@@ -775,6 +782,11 @@ public class DateLiteral extends LiteralExpr {
}
}
+ private boolean isLeapYear() {
+ return ((year % 4) == 0) && ((year % 100 != 0) || ((year % 400) == 0 && year > 0));
+ }
+
+ // Validation check should be same as DateV2Value<T>::is_invalid in BE
@Override
public void checkValueValid() throws AnalysisException {
if (year < 0 || year > 9999) {
@@ -783,8 +795,10 @@ public class DateLiteral extends LiteralExpr {
if (month < 1 || month > 12) {
throw new AnalysisException("DateLiteral has invalid month value: " + month);
}
- if (day < 1 || day > 31) {
- throw new AnalysisException("DateLiteral has invalid day value: " + day);
+ if (day < 1 || day > DAYS_IN_MONTH[(int) month]) {
+ if (!(month == 2 && day == 29 && isLeapYear())) {
+ throw new AnalysisException("DateLiteral has invalid day value: " + day);
+ }
}
if (type.isDatetimeV2() || type.isDatetime()) {
if (hour < 0 || hour > 24) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java
index 02c7b26b25..e7dae9617b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java
@@ -98,7 +98,18 @@ public enum ExpressionFunctions {
FEFunctionInvoker invoker = getFunction(signature);
if (invoker != null) {
try {
- return invoker.invoke(constExpr.getChildrenWithoutCast());
+ if (fn.getReturnType().isDateType()) {
+ Expr dateLiteral = invoker.invoke(constExpr.getChildrenWithoutCast());
+ Preconditions.checkArgument(dateLiteral instanceof DateLiteral);
+ try {
+ ((DateLiteral) dateLiteral).checkValueValid();
+ } catch (AnalysisException e) {
+ return NullLiteral.create(dateLiteral.getType());
+ }
+ return dateLiteral;
+ } else {
+ return invoker.invoke(constExpr.getChildrenWithoutCast());
+ }
} catch (AnalysisException e) {
LOG.debug("failed to invoke", e);
return constExpr;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
index 1094ca6564..62038dec27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
@@ -193,6 +193,11 @@ public class StringLiteral extends LiteralExpr {
throw e;
}
}
+ try {
+ newLiteral.checkValueValid();
+ } catch (AnalysisException e) {
+ return NullLiteral.create(newLiteral.getType());
+ }
return newLiteral;
}
diff --git a/regression-test/data/datatype_p0/date/test_invalid_date.out b/regression-test/data/datatype_p0/date/test_invalid_date.out
new file mode 100644
index 0000000000..80b3c3963d
--- /dev/null
+++ b/regression-test/data/datatype_p0/date/test_invalid_date.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql1 --
+\N
+
+-- !sql2 --
+\N
+
diff --git a/regression-test/suites/datatype_p0/date/test_invalid_date.groovy b/regression-test/suites/datatype_p0/date/test_invalid_date.groovy
new file mode 100644
index 0000000000..6b683d90b6
--- /dev/null
+++ b/regression-test/suites/datatype_p0/date/test_invalid_date.groovy
@@ -0,0 +1,37 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_invalid_date") {
+ def tbName = "test_invalid_date"
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ c0 int,
+ c1 char(10),
+ c2 date,
+ c3 datev2
+ )
+ UNIQUE KEY(c0)
+ DISTRIBUTED BY HASH(c0) BUCKETS 5 properties("replication_num" = "1");
+ """
+ sql "insert into ${tbName} values(1, 'test1', '2000-01-01', '2000-01-01')"
+
+ qt_sql1 "select str_to_date('202301', '%Y%m');"
+ qt_sql2 "select str_to_date('202301', '%Y%m') from ${tbName}"
+ sql "DROP TABLE ${tbName}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/20: [Bug](exec) enable warning on ignoring function return value for vctx (#16157)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 73d6ffc47d42cb2a706120d40d2241570406b021
Author: Pxl <px...@qq.com>
AuthorDate: Sun Jan 29 17:23:21 2023 +0800
[Bug](exec) enable warning on ignoring function return value for vctx (#16157)
* enable warning on ignoring function return value for vctx
---
be/src/vec/core/sort_cursor.h | 6 +++---
be/src/vec/exec/join/process_hash_table_probe_impl.h | 10 +++++++---
be/src/vec/exec/vunion_node.cpp | 3 ++-
be/src/vec/exprs/vexpr_context.h | 15 ++++++++-------
be/src/vec/olap/vcollect_iterator.cpp | 2 +-
be/test/vec/exprs/vexpr_test.cpp | 15 +++++++++++++--
6 files changed, 34 insertions(+), 17 deletions(-)
diff --git a/be/src/vec/core/sort_cursor.h b/be/src/vec/core/sort_cursor.h
index b13316fe40..a2174c3cb5 100644
--- a/be/src/vec/core/sort_cursor.h
+++ b/be/src/vec/core/sort_cursor.h
@@ -232,11 +232,11 @@ struct ReceiveQueueSortCursorImpl : public MergeSortCursorImpl {
bool has_next_block() override {
auto status = _block_supplier(&_block_ptr);
if (status.ok() && _block_ptr != nullptr) {
- for (int i = 0; i < desc.size(); ++i) {
- _ordering_expr[i]->execute(_block_ptr, &desc[i].column_number);
+ for (int i = 0; status.ok() && i < desc.size(); ++i) {
+ status = _ordering_expr[i]->execute(_block_ptr, &desc[i].column_number);
}
MergeSortCursorImpl::reset(*_block_ptr);
- return true;
+ return status.ok();
}
_block_ptr = nullptr;
return false;
diff --git a/be/src/vec/exec/join/process_hash_table_probe_impl.h b/be/src/vec/exec/join/process_hash_table_probe_impl.h
index 05499842cb..54319f782c 100644
--- a/be/src/vec/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/vec/exec/join/process_hash_table_probe_impl.h
@@ -17,6 +17,7 @@
#pragma once
+#include "common/status.h"
#include "process_hash_table_probe.h"
#include "vhash_join_node.h"
@@ -232,9 +233,10 @@ Status ProcessHashTableProbe<JoinOpType>::do_process(HashTableType& hash_table_c
? decltype(key_getter.find_key(hash_table_ctx.hash_table, probe_index,
*_arena)) {nullptr, false}
: key_getter.find_key(hash_table_ctx.hash_table, probe_index, *_arena);
- if (probe_index + PREFETCH_STEP < probe_rows)
+ if (probe_index + PREFETCH_STEP < probe_rows) {
key_getter.template prefetch<true>(hash_table_ctx.hash_table,
probe_index + PREFETCH_STEP, *_arena);
+ }
if constexpr (JoinOpType == TJoinOp::LEFT_ANTI_JOIN ||
JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
@@ -414,9 +416,10 @@ Status ProcessHashTableProbe<JoinOpType>::do_process_with_other_join_conjuncts(
? decltype(key_getter.find_key(hash_table_ctx.hash_table, probe_index,
*_arena)) {nullptr, false}
: key_getter.find_key(hash_table_ctx.hash_table, probe_index, *_arena);
- if (probe_index + PREFETCH_STEP < probe_rows)
+ if (probe_index + PREFETCH_STEP < probe_rows) {
key_getter.template prefetch<true>(hash_table_ctx.hash_table,
probe_index + PREFETCH_STEP, *_arena);
+ }
if (find_result.is_found()) {
auto& mapped = find_result.get_mapped();
auto origin_offset = current_offset;
@@ -493,7 +496,8 @@ Status ProcessHashTableProbe<JoinOpType>::do_process_with_other_join_conjuncts(
if (output_block->rows()) {
int result_column_id = -1;
int orig_columns = output_block->columns();
- (*_join_node->_vother_join_conjunct_ptr)->execute(output_block, &result_column_id);
+ RETURN_IF_ERROR((*_join_node->_vother_join_conjunct_ptr)
+ ->execute(output_block, &result_column_id));
auto column = output_block->get_by_position(result_column_id).column;
if constexpr (JoinOpType == TJoinOp::LEFT_OUTER_JOIN ||
diff --git a/be/src/vec/exec/vunion_node.cpp b/be/src/vec/exec/vunion_node.cpp
index e3343e25de..a18bb1215e 100644
--- a/be/src/vec/exec/vunion_node.cpp
+++ b/be/src/vec/exec/vunion_node.cpp
@@ -280,7 +280,8 @@ Block VUnionNode::materialize_block(Block* src_block) {
ColumnsWithTypeAndName colunms;
for (size_t i = 0; i < child_exprs.size(); ++i) {
int result_column_id = -1;
- child_exprs[i]->execute(src_block, &result_column_id);
+ auto state = child_exprs[i]->execute(src_block, &result_column_id);
+ CHECK(state.ok()) << state.to_string();
colunms.emplace_back(src_block->get_by_position(result_column_id));
}
_child_row_idx += src_block->rows();
diff --git a/be/src/vec/exprs/vexpr_context.h b/be/src/vec/exprs/vexpr_context.h
index b454833ed3..82ff9d11b8 100644
--- a/be/src/vec/exprs/vexpr_context.h
+++ b/be/src/vec/exprs/vexpr_context.h
@@ -28,11 +28,11 @@ class VExprContext {
public:
VExprContext(VExpr* expr);
~VExprContext();
- Status prepare(RuntimeState* state, const RowDescriptor& row_desc);
- Status open(RuntimeState* state);
+ [[nodiscard]] Status prepare(RuntimeState* state, const RowDescriptor& row_desc);
+ [[nodiscard]] Status open(RuntimeState* state);
void close(RuntimeState* state);
- Status clone(RuntimeState* state, VExprContext** new_ctx);
- Status execute(Block* block, int* result_column_id);
+ [[nodiscard]] Status clone(RuntimeState* state, VExprContext** new_ctx);
+ [[nodiscard]] Status execute(Block* block, int* result_column_id);
VExpr* root() { return _root; }
void set_root(VExpr* expr) { _root = expr; }
@@ -53,9 +53,10 @@ public:
return _fn_contexts[i];
}
- static Status filter_block(VExprContext* vexpr_ctx, Block* block, int column_to_keep);
- static Status filter_block(const std::unique_ptr<VExprContext*>& vexpr_ctx_ptr, Block* block,
- int column_to_keep);
+ [[nodiscard]] static Status filter_block(VExprContext* vexpr_ctx, Block* block,
+ int column_to_keep);
+ [[nodiscard]] static Status filter_block(const std::unique_ptr<VExprContext*>& vexpr_ctx_ptr,
+ Block* block, int column_to_keep);
static Block get_output_block_after_execute_exprs(const std::vector<vectorized::VExprContext*>&,
const Block&, Status&);
diff --git a/be/src/vec/olap/vcollect_iterator.cpp b/be/src/vec/olap/vcollect_iterator.cpp
index cc07e9be30..cbcedb878a 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -137,7 +137,7 @@ Status VCollectIterator::build_heap(std::vector<RowsetReaderSharedPtr>& rs_reade
if (!s.ok()) {
delete (*iter);
iter = _children.erase(iter);
- if (!s.is<END_OF_FILE>()) {
+ if (!s.is_end_of_file()) {
return s;
}
} else {
diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp
index 593cfc932e..ca8f8938a8 100644
--- a/be/test/vec/exprs/vexpr_test.cpp
+++ b/be/test/vec/exprs/vexpr_test.cpp
@@ -71,8 +71,11 @@ TEST(TEST_VEXPR, ABSTEST) {
doris::TQueryGlobals(), nullptr);
runtime_stat.init_mem_trackers();
runtime_stat.set_desc_tbl(desc_tbl);
- context->prepare(&runtime_stat, row_desc);
- context->open(&runtime_stat);
+ auto state = doris::Status::OK();
+ state = context->prepare(&runtime_stat, row_desc);
+ ASSERT_TRUE(state.ok());
+ state = context->open(&runtime_stat);
+ ASSERT_TRUE(state.ok());
auto block = row_batch.convert_to_vec_block();
int ts = -1;
@@ -116,12 +119,20 @@ TEST(TEST_VEXPR, ABSTEST2) {
DescriptorTbl desc_tbl;
desc_tbl._slot_desc_map[0] = tuple_desc->slots()[0];
runtime_stat.set_desc_tbl(&desc_tbl);
+<<<<<<< HEAD
context->prepare(&runtime_stat, row_desc);
context->open(&runtime_stat);
auto block = row_batch.convert_to_vec_block();
int ts = -1;
context->execute(&block, &ts);
+=======
+ auto state = Status::OK();
+ state = context->prepare(&runtime_stat, row_desc);
+ ASSERT_TRUE(state.ok());
+ state = context->open(&runtime_stat);
+ ASSERT_TRUE(state.ok());
+>>>>>>> 46347a51d2... [Bug](exec) enable warning on ignoring function return value for vctx (#16157)
context->close(&runtime_stat);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 13/20: [fix](olap) Incorrect reserving size for PredicateColumn converted from ColumnDictionary (#16249)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5cf69f302b354e2d0dbb8dc4422c0c11b22174fa
Author: Jerry Hu <mr...@gmail.com>
AuthorDate: Mon Jan 30 20:28:22 2023 +0800
[fix](olap) Incorrect reserving size for PredicateColumn converted from ColumnDictionary (#16249)
---
be/src/vec/columns/column_dictionary.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
index 95cd848763..2812c07103 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -293,7 +293,7 @@ public:
convert_dict_codes_if_necessary();
}
auto res = vectorized::PredicateColumnType<TYPE_STRING>::create();
- res->reserve(_codes.size());
+ res->reserve(_codes.capacity());
for (size_t i = 0; i < _codes.size(); ++i) {
auto& code = reinterpret_cast<T&>(_codes[i]);
auto value = _dict.get_value(code);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/20: [Enhancement](profile) use queryId of originStatement as the queryId. (#15898)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 691baff248dc8d79eb3c019c5f05cd4543169959
Author: wxy <du...@gmail.com>
AuthorDate: Sun Jan 15 18:26:14 2023 +0800
[Enhancement](profile) use queryId of originStatement as the queryId. (#15898)
* [Enhancement](profile) use queryId of originStatement as the queryId.
Co-authored-by: wangxiangyu@360shuke.com <wa...@360shuke.com>
---
.../apache/doris/analysis/ShowQueryProfileStmt.java | 1 +
.../apache/doris/common/util/ProfileManager.java | 21 ++++++++++++++-------
.../main/java/org/apache/doris/load/ExportJob.java | 9 ++++++++-
.../org/apache/doris/load/loadv2/BrokerLoadJob.java | 3 ++-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 5 +++++
.../main/java/org/apache/doris/qe/StmtExecutor.java | 1 +
.../org/apache/doris/task/ExportExportingTask.java | 3 ++-
7 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
index 15be652aed..b36cd30159 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
@@ -34,6 +34,7 @@ public class ShowQueryProfileStmt extends ShowStmt {
// This should be same as ProfileManager.PROFILE_HEADERS
public static final ShowResultSetMetaData META_DATA_QUERY_IDS =
ShowResultSetMetaData.builder()
+ .addColumn(new Column("JobId", ScalarType.createVarchar(128)))
.addColumn(new Column("QueryId", ScalarType.createVarchar(128)))
.addColumn(new Column("User", ScalarType.createVarchar(128)))
.addColumn(new Column("DefaultDb", ScalarType.createVarchar(128)))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
index 3922c496c7..c569d10c82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
@@ -58,6 +58,8 @@ public class ProfileManager {
private static volatile ProfileManager INSTANCE = null;
// private static final int ARRAY_SIZE = 100;
// private static final int TOTAL_LEN = 1000 * ARRAY_SIZE ;
+ // just use for load profile and export profile
+ public static final String JOB_ID = "Job ID";
public static final String QUERY_ID = "Query ID";
public static final String START_TIME = "Start Time";
public static final String END_TIME = "End Time";
@@ -82,7 +84,7 @@ public class ProfileManager {
}
public static final ArrayList<String> PROFILE_HEADERS = new ArrayList(
- Arrays.asList(QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
+ Arrays.asList(JOB_ID, QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
START_TIME, END_TIME, TOTAL_TIME, QUERY_STATE, TRACE_ID));
private class ProfileElement {
@@ -146,25 +148,26 @@ public class ProfileManager {
}
ProfileElement element = createElement(profile);
- String queryId = element.infoStrings.get(ProfileManager.QUERY_ID);
+ String key = isQueryProfile(profile) ? element.infoStrings.get(ProfileManager.QUERY_ID)
+ : element.infoStrings.get(ProfileManager.JOB_ID);
// check when push in, which can ensure every element in the list has QUERY_ID column,
// so there is no need to check when remove element from list.
- if (Strings.isNullOrEmpty(queryId)) {
+ if (Strings.isNullOrEmpty(key)) {
LOG.warn("the key or value of Map is null, "
- + "may be forget to insert 'QUERY_ID' column into infoStrings");
+ + "may be forget to insert 'QUERY_ID' or 'JOB_ID' column into infoStrings");
}
// a profile may be updated multiple times in queryIdToProfileMap,
// and only needs to be inserted into the queryIdDeque for the first time.
- queryIdToProfileMap.put(queryId, element);
+ queryIdToProfileMap.put(key, element);
writeLock.lock();
try {
- if (!queryIdDeque.contains(queryId)) {
+ if (!queryIdDeque.contains(key)) {
if (queryIdDeque.size() >= Config.max_query_profile_num) {
queryIdToProfileMap.remove(queryIdDeque.getFirst());
queryIdDeque.removeFirst();
}
- queryIdDeque.addLast(queryId);
+ queryIdDeque.addLast(key);
}
} finally {
writeLock.unlock();
@@ -331,4 +334,8 @@ public class ProfileManager {
readLock.unlock();
}
}
+
+ public boolean isQueryProfile(RuntimeProfile profile) {
+ return "Query".equals(profile.getName());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 7c68e24689..a7f6bef921 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -51,6 +51,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.planner.DataPartition;
@@ -117,6 +118,7 @@ public class ExportJob implements Writable {
}
private long id;
+ private String queryId;
private String label;
private long dbId;
private long tableId;
@@ -176,6 +178,7 @@ public class ExportJob implements Writable {
public ExportJob() {
this.id = -1;
+ this.queryId = "";
this.dbId = -1;
this.tableId = -1;
this.state = JobState.PENDING;
@@ -202,11 +205,11 @@ public class ExportJob implements Writable {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
Preconditions.checkNotNull(stmt.getBrokerDesc());
this.brokerDesc = stmt.getBrokerDesc();
-
this.columnSeparator = stmt.getColumnSeparator();
this.lineDelimiter = stmt.getLineDelimiter();
this.properties = stmt.getProperties();
this.label = this.properties.get(ExportStmt.LABEL);
+ this.queryId = ConnectContext.get() != null ? DebugUtil.printId(ConnectContext.get().queryId()) : "N/A";
String path = stmt.getPath();
Preconditions.checkArgument(!Strings.isNullOrEmpty(path));
@@ -744,6 +747,10 @@ public class ExportJob implements Writable {
return label;
}
+ public String getQueryId() {
+ return queryId;
+ }
+
@Override
public String toString() {
return "ExportJob [jobId=" + id
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index c396297afd..5d42fff639 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -316,7 +316,8 @@ public class BrokerLoadJob extends BulkLoadJob {
}
RuntimeProfile summaryProfile = new RuntimeProfile("Summary");
- summaryProfile.addInfoString(ProfileManager.QUERY_ID, String.valueOf(id));
+ summaryProfile.addInfoString(ProfileManager.JOB_ID, String.valueOf(this.id));
+ summaryProfile.addInfoString(ProfileManager.QUERY_ID, this.queryId);
summaryProfile.addInfoString(ProfileManager.START_TIME, TimeUtils.longToTimeString(createTimestamp));
summaryProfile.addInfoString(ProfileManager.END_TIME, TimeUtils.longToTimeString(finishTimestamp));
summaryProfile.addInfoString(ProfileManager.TOTAL_TIME,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
index 07a43fc6e5..a394a62d4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
@@ -31,6 +31,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.LogBuilder;
import org.apache.doris.common.util.LogKey;
import org.apache.doris.common.util.SqlParserUtils;
@@ -74,6 +75,8 @@ public abstract class BulkLoadJob extends LoadJob {
// input params
protected BrokerDesc brokerDesc;
+ // queryId of OriginStatement
+ protected String queryId;
// this param is used to persist the expr of columns
// the origin stmt is persisted instead of columns expr
// the expr of columns will be reanalyze when the log is replayed
@@ -101,9 +104,11 @@ public abstract class BulkLoadJob extends LoadJob {
this.userInfo = userInfo;
if (ConnectContext.get() != null) {
+ this.queryId = DebugUtil.printId(ConnectContext.get().queryId());
SessionVariable var = ConnectContext.get().getSessionVariable();
sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
} else {
+ this.queryId = "N/A";
sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index e42546b9b9..30c5e1dbe4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -301,6 +301,7 @@ public class StmtExecutor implements ProfileWriter {
private Map<String, String> getSummaryInfo() {
Map<String, String> infos = Maps.newLinkedHashMap();
+ infos.put(ProfileManager.JOB_ID, "N/A");
infos.put(ProfileManager.QUERY_ID, DebugUtil.printId(context.queryId()));
infos.put(ProfileManager.QUERY_TYPE, queryType);
infos.put(ProfileManager.DORIS_VERSION, Version.DORIS_BUILD_VERSION);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
index 8066e280a5..fe6dde27fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
@@ -252,7 +252,8 @@ public class ExportExportingTask extends MasterTask {
private void initProfile() {
profile = new RuntimeProfile("ExportJob");
RuntimeProfile summaryProfile = new RuntimeProfile("Summary");
- summaryProfile.addInfoString(ProfileManager.QUERY_ID, String.valueOf(job.getId()));
+ summaryProfile.addInfoString(ProfileManager.JOB_ID, String.valueOf(job.getId()));
+ summaryProfile.addInfoString(ProfileManager.QUERY_ID, job.getQueryId());
summaryProfile.addInfoString(ProfileManager.START_TIME, TimeUtils.longToTimeString(job.getStartTimeMs()));
long currentTimestamp = System.currentTimeMillis();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 12/20: [fix](vresultsink) BufferControlBlock may block all fragment handle threads (#16231)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit eadd33a6094aead5de4d95010a8f654d0fe18018
Author: chenlinzhong <49...@qq.com>
AuthorDate: Mon Jan 30 16:53:21 2023 +0800
[fix](vresultsink) BufferControlBlock may block all fragment handle threads (#16231)
BufferControlBlock may block all fragment handle threads leads to be out of work
modify include:
BufferControlBlock cancel after max timeout
StmtExcutor notify be to cancel the fragment when unexcepted occur
more details see issue #16203
---
be/src/runtime/result_buffer_mgr.cpp | 10 +++++++++-
be/src/runtime/result_buffer_mgr.h | 3 ++-
be/src/runtime/result_file_sink.cpp | 2 +-
be/src/runtime/result_sink.cpp | 2 +-
be/src/runtime/runtime_state.h | 1 +
be/src/vec/sink/vresult_file_sink.cpp | 3 ++-
be/src/vec/sink/vresult_sink.cpp | 5 +++--
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 6 +++++-
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java | 6 ++++++
9 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/be/src/runtime/result_buffer_mgr.cpp b/be/src/runtime/result_buffer_mgr.cpp
index ce589dd745..b9d6cc7234 100644
--- a/be/src/runtime/result_buffer_mgr.cpp
+++ b/be/src/runtime/result_buffer_mgr.cpp
@@ -58,7 +58,8 @@ Status ResultBufferMgr::init() {
}
Status ResultBufferMgr::create_sender(const TUniqueId& query_id, int buffer_size,
- std::shared_ptr<BufferControlBlock>* sender) {
+ std::shared_ptr<BufferControlBlock>* sender,
+ int query_timeout) {
*sender = find_control_block(query_id);
if (*sender != nullptr) {
LOG(WARNING) << "already have buffer control block for this instance " << query_id;
@@ -70,6 +71,13 @@ Status ResultBufferMgr::create_sender(const TUniqueId& query_id, int buffer_size
{
std::lock_guard<std::mutex> l(_lock);
_buffer_map.insert(std::make_pair(query_id, control_block));
+ // BufferControlBlock should destroy after max_timeout
+ // for exceed max_timeout FE will return timeout to client
+ // otherwise in some case may block all fragment handle threads
+ // details see issue https://github.com/apache/doris/issues/16203
+ // add extra 5s for avoid corner case
+ int64_t max_timeout = time(nullptr) + query_timeout + 5;
+ cancel_at_time(max_timeout, query_id);
}
*sender = control_block;
return Status::OK();
diff --git a/be/src/runtime/result_buffer_mgr.h b/be/src/runtime/result_buffer_mgr.h
index 26a07bd90c..cc16e771a0 100644
--- a/be/src/runtime/result_buffer_mgr.h
+++ b/be/src/runtime/result_buffer_mgr.h
@@ -47,7 +47,8 @@ public:
// the returned sender do not need release
// sender is not used when call cancel or unregister
Status create_sender(const TUniqueId& query_id, int buffer_size,
- std::shared_ptr<BufferControlBlock>* sender);
+ std::shared_ptr<BufferControlBlock>* sender,
+ int query_timeout);
// fetch data, used by RPC
Status fetch_data(const TUniqueId& fragment_id, TFetchDataResult* result);
diff --git a/be/src/runtime/result_file_sink.cpp b/be/src/runtime/result_file_sink.cpp
index cd3e61659a..e294def28f 100644
--- a/be/src/runtime/result_file_sink.cpp
+++ b/be/src/runtime/result_file_sink.cpp
@@ -100,7 +100,7 @@ Status ResultFileSink::prepare(RuntimeState* state) {
if (_is_top_sink) {
// create sender
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
- state->fragment_instance_id(), _buf_size, &_sender));
+ state->fragment_instance_id(), _buf_size, &_sender, state->query_timeout()));
// create writer
_writer.reset(new (std::nothrow) FileResultWriter(
_file_opts.get(), _storage_type, state->fragment_instance_id(), _output_expr_ctxs,
diff --git a/be/src/runtime/result_sink.cpp b/be/src/runtime/result_sink.cpp
index de393e9e11..a19c44ea7c 100644
--- a/be/src/runtime/result_sink.cpp
+++ b/be/src/runtime/result_sink.cpp
@@ -67,7 +67,7 @@ Status ResultSink::prepare(RuntimeState* state) {
// create sender
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(state->fragment_instance_id(),
- _buf_size, &_sender));
+ _buf_size, &_sender, state->query_timeout()));
// create writer based on sink type
switch (_sink_type) {
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index ca6dc42c4e..cdbfeff5fa 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -97,6 +97,7 @@ public:
return _query_options.abort_on_default_limit_exceeded;
}
int max_errors() const { return _query_options.max_errors; }
+ int query_timeout() const { return _query_options.query_timeout; }
int max_io_buffers() const { return _query_options.max_io_buffers; }
int num_scanner_threads() const { return _query_options.num_scanner_threads; }
TQueryType::type query_type() const { return _query_options.query_type; }
diff --git a/be/src/vec/sink/vresult_file_sink.cpp b/be/src/vec/sink/vresult_file_sink.cpp
index 7bfc8b4c8a..948dfce4d0 100644
--- a/be/src/vec/sink/vresult_file_sink.cpp
+++ b/be/src/vec/sink/vresult_file_sink.cpp
@@ -103,7 +103,8 @@ Status VResultFileSink::prepare(RuntimeState* state) {
if (_is_top_sink) {
// create sender
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
- state->fragment_instance_id(), _buf_size, &_sender));
+ state->fragment_instance_id(), _buf_size, &_sender,
+ state->query_timeout()));
// create writer
_writer.reset(new (std::nothrow) VFileResultWriter(
_file_opts.get(), _storage_type, state->fragment_instance_id(), _output_vexpr_ctxs,
diff --git a/be/src/vec/sink/vresult_sink.cpp b/be/src/vec/sink/vresult_sink.cpp
index 77e64a8959..5d3fe0ec9a 100644
--- a/be/src/vec/sink/vresult_sink.cpp
+++ b/be/src/vec/sink/vresult_sink.cpp
@@ -61,8 +61,9 @@ Status VResultSink::prepare(RuntimeState* state) {
RETURN_IF_ERROR(prepare_exprs(state));
// create sender
- RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(state->fragment_instance_id(),
- _buf_size, &_sender));
+ RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
+ state->fragment_instance_id(), _buf_size, &_sender,
+ state->query_timeout()));
// create writer based on sink type
switch (_sink_type) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 7d32530eb6..40b085505d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -940,6 +940,10 @@ public class Coordinator {
// fragment,
// if any, as well as all plan fragments on remote nodes.
public void cancel() {
+ cancel(Types.PPlanFragmentCancelReason.USER_CANCEL);
+ }
+
+ public void cancel(Types.PPlanFragmentCancelReason cancelReason) {
lock();
try {
if (!queryStatus.ok()) {
@@ -949,7 +953,7 @@ public class Coordinator {
queryStatus.setStatus(Status.CANCELLED);
}
LOG.warn("cancel execution of query, this is outside invoke");
- cancelInternal(Types.PPlanFragmentCancelReason.USER_CANCEL);
+ cancelInternal(cancelReason);
} finally {
unlock();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 30c5e1dbe4..55e6832794 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -100,6 +100,7 @@ import org.apache.doris.planner.Planner;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.proto.Data;
import org.apache.doris.proto.InternalService;
+import org.apache.doris.proto.Types;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.cache.Cache;
import org.apache.doris.qe.cache.CacheAnalyzer;
@@ -1205,6 +1206,11 @@ public class StmtExecutor implements ProfileWriter {
context.getState().setEof();
plannerProfile.setQueryFetchResultFinishTime();
} catch (Exception e) {
+ // notify all be cancel runing fragment
+ // in some case may block all fragment handle threads
+ // details see issue https://github.com/apache/doris/issues/16203
+ LOG.warn("cancel fragment query_id:{} cause {}", DebugUtil.printId(context.queryId()), e.getMessage());
+ coord.cancel(Types.PPlanFragmentCancelReason.INTERNAL_ERROR);
fetchResultSpan.recordException(e);
throw e;
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 14/20: [fix](planner) Pushdown constant predicate to all scan node in the lieteral view. #16217
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cb47682d09f18bd0e7c8e4bf1643323579dabcb3
Author: AKIRA <33...@users.noreply.github.com>
AuthorDate: Mon Jan 30 22:18:43 2023 +0800
[fix](planner) Pushdown constant predicate to all scan node in the lieteral view. #16217
Before this PR, planner might push a constant FALSE predicate to the wrong scan nodes in the literal view, and make this predicate useless
---
.../apache/doris/planner/SingleNodePlanner.java | 23 ++----
.../data/query_p0/literal_view/lietral_test.out | 3 +
.../query_p0/literal_view/lietral_test.groovy | 88 ++++++++++++++++++++++
3 files changed, 96 insertions(+), 18 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index b5f50f9317..ee5f26b173 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -1769,7 +1769,7 @@ public class SingleNodePlanner {
return;
}
- final List<Expr> newConjuncts = cloneExprs(conjuncts);
+ List<Expr> newConjuncts = cloneExprs(conjuncts);
final QueryStmt stmt = inlineViewRef.getViewStmt();
final Analyzer viewAnalyzer = inlineViewRef.getAnalyzer();
viewAnalyzer.markConjunctsAssigned(conjuncts);
@@ -1778,23 +1778,10 @@ public class SingleNodePlanner {
if (select.getAggInfo() != null) {
viewAnalyzer.registerConjuncts(newConjuncts, select.getAggInfo().getOutputTupleId().asList());
} else if (select.getTableRefs().size() > 1) {
- // Conjuncts will be assigned to the lowest outer join node or non-outer join's leaf children.
- for (int i = select.getTableRefs().size(); i > 1; i--) {
- final TableRef joinInnerChild = select.getTableRefs().get(i - 1);
- final TableRef joinOuterChild = select.getTableRefs().get(i - 2);
- if (!joinInnerChild.getJoinOp().isOuterJoin()) {
- // lowest join isn't outer join.
- if (i == 2) {
- // Register constant for inner.
- viewAnalyzer.registerConjuncts(newConjuncts, joinInnerChild.getDesc().getId().asList());
- // Register constant for outer.
- final List<Expr> cloneConjuncts = cloneExprs(newConjuncts);
- viewAnalyzer.registerConjuncts(cloneConjuncts, joinOuterChild.getDesc().getId().asList());
- }
- continue;
- }
- viewAnalyzer.registerConjuncts(newConjuncts, joinOuterChild.getId());
- break;
+ for (int i = select.getTableRefs().size() - 1; i >= 0; i--) {
+ viewAnalyzer.registerConjuncts(newConjuncts,
+ select.getTableRefs().get(i).getDesc().getId().asList());
+ newConjuncts = cloneExprs(newConjuncts);
}
} else {
Preconditions.checkArgument(select.getTableRefs().size() == 1);
diff --git a/regression-test/data/query_p0/literal_view/lietral_test.out b/regression-test/data/query_p0/literal_view/lietral_test.out
new file mode 100644
index 0000000000..9c9c4c6c8a
--- /dev/null
+++ b/regression-test/data/query_p0/literal_view/lietral_test.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+
diff --git a/regression-test/suites/query_p0/literal_view/lietral_test.groovy b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
new file mode 100644
index 0000000000..1db3f6d720
--- /dev/null
+++ b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("literal_view_test") {
+
+ sql """DROP TABLE IF EXISTS table1"""
+
+ sql """
+ CREATE table table1(
+ `a` varchar(150) NULL COMMENT "",
+ `b` varchar(60) NULL COMMENT ""
+ )ENGINE=OLAP
+ UNIQUE KEY(`a`, `b`)
+ DISTRIBUTED BY HASH(`b`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql """
+ INSERT into table1
+ values('org1','code1');
+ """
+
+ sql """DROP TABLE IF EXISTS table2"""
+
+ sql """
+ CREATE table table2(
+ `c` varchar(40) NOT NULL COMMENT "c"
+ )ENGINE=OLAP
+ UNIQUE KEY(`c`)
+ DISTRIBUTED BY HASH(`c`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+
+ """
+
+ sql """DROP TABLE IF EXISTS table3"""
+
+ sql """
+ CREATE table table3 (
+ `c` varchar(40) NOT NULL COMMENT "c"
+ )ENGINE=OLAP
+ UNIQUE KEY(`c`)
+ DISTRIBUTED BY HASH(`c`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql """DROP VIEW IF EXISTS `test_v`"""
+
+ sql """
+ CREATE view `test_v` as
+ select t1.b
+ from table1 as t1
+ left outer JOIN table2 as org ON t1.a = org.c
+ left outer join table3 as doi on t1.a = doi.c
+ ;
+ """
+
+ qt_sql """
+ SELECT b
+ FROM test_v
+ WHERE substring('2022-12',6,2)='01';
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 10/20: [improvement](metrics) Metrics add all rowset nums and segment nums (#16208)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit df5ff77587ef471e3a685405d21f88ab16f3a427
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Mon Jan 30 09:55:32 2023 +0800
[improvement](metrics) Metrics add all rowset nums and segment nums (#16208)
---
be/src/common/daemon.cpp | 6 ++++++
be/src/olap/tablet.h | 11 +++++++++++
be/src/olap/tablet_manager.cpp | 22 ++++++++++++++++++++++
be/src/olap/tablet_manager.h | 3 +++
be/src/util/doris_metrics.cpp | 6 ++++++
be/src/util/doris_metrics.h | 3 +++
6 files changed, 51 insertions(+)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 149b2f1ce4..2f6afb99b7 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -49,6 +49,7 @@
#include "exprs/utility_functions.h"
#include "geo/geo_functions.h"
#include "olap/options.h"
+#include "olap/storage_engine.h"
#include "runtime/bufferpool/buffer_pool.h"
#include "runtime/exec_env.h"
#include "runtime/fragment_mgr.h"
@@ -336,6 +337,11 @@ void Daemon::calculate_metrics_thread() {
DorisMetrics::instance()->system_metrics()->get_network_traffic(
&lst_net_send_bytes, &lst_net_receive_bytes);
}
+
+ DorisMetrics::instance()->all_rowset_nums->set_value(
+ StorageEngine::instance()->tablet_manager()->get_rowset_nums());
+ DorisMetrics::instance()->all_segment_nums->set_value(
+ StorageEngine::instance()->tablet_manager()->get_segment_nums());
}
} while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(15)));
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index dcbb0097fc..4b444f13ec 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -97,6 +97,7 @@ public:
size_t num_rows();
int version_count() const;
+ uint64_t segment_count() const;
Version max_version() const;
Version max_version_unlocked() const;
CumulativeCompactionPolicy* cumulative_compaction_policy();
@@ -563,6 +564,7 @@ inline size_t Tablet::num_rows() {
}
inline int Tablet::version_count() const {
+ std::shared_lock rdlock(_meta_lock);
return _tablet_meta->version_count();
}
@@ -570,6 +572,15 @@ inline Version Tablet::max_version() const {
return _tablet_meta->max_version();
}
+inline uint64_t Tablet::segment_count() const {
+ std::shared_lock rdlock(_meta_lock);
+ uint64_t segment_nums = 0;
+ for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ segment_nums += rs_meta->num_segments();
+ }
+ return segment_nums;
+}
+
inline Version Tablet::max_version_unlocked() const {
return _tablet_meta->max_version();
}
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 1a5183b080..23c3e7c72c 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -598,6 +598,28 @@ std::vector<TabletSharedPtr> TabletManager::get_all_tablet() {
return res;
}
+uint64_t TabletManager::get_rowset_nums() {
+ uint64_t rowset_nums = 0;
+ for (const auto& tablets_shard : _tablets_shards) {
+ std::shared_lock rdlock(tablets_shard.lock);
+ for (const auto& tablet_map : tablets_shard.tablet_map) {
+ rowset_nums += tablet_map.second->version_count();
+ }
+ }
+ return rowset_nums;
+}
+
+uint64_t TabletManager::get_segment_nums() {
+ uint64_t segment_nums = 0;
+ for (const auto& tablets_shard : _tablets_shards) {
+ std::shared_lock rdlock(tablets_shard.lock);
+ for (const auto& tablet_map : tablets_shard.tablet_map) {
+ segment_nums += tablet_map.second->segment_count();
+ }
+ }
+ return segment_nums;
+}
+
bool TabletManager::get_tablet_id_and_schema_hash_from_path(const string& path,
TTabletId* tablet_id,
TSchemaHash* schema_hash) {
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 75c29bdac7..bbb298cae2 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -80,6 +80,9 @@ public:
std::vector<TabletSharedPtr> get_all_tablet();
+ uint64_t get_rowset_nums();
+ uint64_t get_segment_nums();
+
// Extract tablet_id and schema_hash from given path.
//
// The normal path pattern is like "/data/{shard_id}/{tablet_id}/{schema_hash}/xxx.data".
diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp
index 6f61ef7c36..586a66fbb9 100644
--- a/be/src/util/doris_metrics.cpp
+++ b/be/src/util/doris_metrics.cpp
@@ -136,6 +136,9 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(process_fd_num_limit_hard, MetricUnit::NOUNIT
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_cumulative_max_compaction_score, MetricUnit::NOUNIT);
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_base_max_compaction_score, MetricUnit::NOUNIT);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_rowset_nums, MetricUnit::NOUNIT);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_segment_nums, MetricUnit::NOUNIT);
+
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_used_permits, MetricUnit::NOUNIT);
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_waitting_permits, MetricUnit::NOUNIT);
@@ -259,6 +262,9 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score);
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score);
+ INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_rowset_nums);
+ INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_segment_nums);
+
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_used_permits);
INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_waitting_permits);
diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h
index d117456d01..300c35c9ec 100644
--- a/be/src/util/doris_metrics.h
+++ b/be/src/util/doris_metrics.h
@@ -137,6 +137,9 @@ public:
IntGauge* tablet_cumulative_max_compaction_score;
IntGauge* tablet_base_max_compaction_score;
+ IntGauge* all_rowset_nums;
+ IntGauge* all_segment_nums;
+
// permits have been used for all compaction tasks
IntGauge* compaction_used_permits;
// permits required by the compaction task which is waiting for permits
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 19/20: [branch-1.2] format code after cherry-pick
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 60f22608a442d203888ea1b3f84205b37b16eab5
Author: morningman <mo...@163.com>
AuthorDate: Tue Jan 31 20:42:34 2023 +0800
[branch-1.2] format code after cherry-pick
---
be/src/runtime/result_buffer_mgr.h | 3 +--
be/src/runtime/result_sink.cpp | 4 ++--
be/src/vec/sink/vresult_file_sink.cpp | 3 +--
be/src/vec/sink/vresult_sink.cpp | 3 +--
4 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/be/src/runtime/result_buffer_mgr.h b/be/src/runtime/result_buffer_mgr.h
index cc16e771a0..c74a078466 100644
--- a/be/src/runtime/result_buffer_mgr.h
+++ b/be/src/runtime/result_buffer_mgr.h
@@ -47,8 +47,7 @@ public:
// the returned sender do not need release
// sender is not used when call cancel or unregister
Status create_sender(const TUniqueId& query_id, int buffer_size,
- std::shared_ptr<BufferControlBlock>* sender,
- int query_timeout);
+ std::shared_ptr<BufferControlBlock>* sender, int query_timeout);
// fetch data, used by RPC
Status fetch_data(const TUniqueId& fragment_id, TFetchDataResult* result);
diff --git a/be/src/runtime/result_sink.cpp b/be/src/runtime/result_sink.cpp
index a19c44ea7c..dfe25a4980 100644
--- a/be/src/runtime/result_sink.cpp
+++ b/be/src/runtime/result_sink.cpp
@@ -66,8 +66,8 @@ Status ResultSink::prepare(RuntimeState* state) {
RETURN_IF_ERROR(prepare_exprs(state));
// create sender
- RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(state->fragment_instance_id(),
- _buf_size, &_sender, state->query_timeout()));
+ RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
+ state->fragment_instance_id(), _buf_size, &_sender, state->query_timeout()));
// create writer based on sink type
switch (_sink_type) {
diff --git a/be/src/vec/sink/vresult_file_sink.cpp b/be/src/vec/sink/vresult_file_sink.cpp
index 948dfce4d0..d1d2cfaa72 100644
--- a/be/src/vec/sink/vresult_file_sink.cpp
+++ b/be/src/vec/sink/vresult_file_sink.cpp
@@ -103,8 +103,7 @@ Status VResultFileSink::prepare(RuntimeState* state) {
if (_is_top_sink) {
// create sender
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
- state->fragment_instance_id(), _buf_size, &_sender,
- state->query_timeout()));
+ state->fragment_instance_id(), _buf_size, &_sender, state->query_timeout()));
// create writer
_writer.reset(new (std::nothrow) VFileResultWriter(
_file_opts.get(), _storage_type, state->fragment_instance_id(), _output_vexpr_ctxs,
diff --git a/be/src/vec/sink/vresult_sink.cpp b/be/src/vec/sink/vresult_sink.cpp
index 5d3fe0ec9a..a5868ce2b0 100644
--- a/be/src/vec/sink/vresult_sink.cpp
+++ b/be/src/vec/sink/vresult_sink.cpp
@@ -62,8 +62,7 @@ Status VResultSink::prepare(RuntimeState* state) {
// create sender
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
- state->fragment_instance_id(), _buf_size, &_sender,
- state->query_timeout()));
+ state->fragment_instance_id(), _buf_size, &_sender, state->query_timeout()));
// create writer based on sink type
switch (_sink_type) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/20: [docker](iceberg) add iceberg docker compose and modify scripts (#16175)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a30881d77032210db45a260e605bbc979b6d5d50
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Sun Jan 29 14:31:27 2023 +0800
[docker](iceberg) add iceberg docker compose and modify scripts (#16175)
Add iceberg docker compose
Rename start-thirdparties-docker.sh to run-thirdparties-docker.sh and support start to stop specified components.
---
.licenserc.yaml | 1 +
docker/thirdparties/custom_settings.env | 23 ++
.../elasticsearch/{es.yaml => es.yaml.tpl} | 0
...hadoop-hive.env.tpl => hadoop-hive.env.tpl.tpl} | 0
.../hive/{hive-2x.yaml => hive-2x.yaml.tpl} | 0
.../docker-compose/iceberg/entrypoint.sh.tpl | 34 +++
.../docker-compose/iceberg/iceberg.env | 24 ++
.../docker-compose/iceberg/iceberg.yaml.tpl | 96 ++++++++
.../docker-compose/iceberg/spark-defaults.conf.tpl | 11 +
.../mysql/{mysql-5.7.yaml => mysql-5.7.yaml.tpl} | 0
.../oracle/{oracle-11.yaml => oracle-11.yaml.tpl} | 2 +-
.../{postgresql-14.yaml => postgresql-14.yaml.tpl} | 0
.../{sqlserver.yaml => sqlserver.yaml.tpl} | 0
docker/thirdparties/run-thirdparties-docker.sh | 251 +++++++++++++++++++++
docker/thirdparties/start-thirdparties-docker.sh | 79 -------
docker/thirdparties/stop-thirdparties-docker.sh | 43 ----
.../developer-guide/regression-testing.md | 111 +++++++--
17 files changed, 528 insertions(+), 147 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 020ee7b4e8..a6038567df 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -77,5 +77,6 @@ header:
- "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql"
- "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql"
- "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/"
+ - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
comment: on-failure
diff --git a/docker/thirdparties/custom_settings.env b/docker/thirdparties/custom_settings.env
new file mode 100644
index 0000000000..d1f9fbb628
--- /dev/null
+++ b/docker/thirdparties/custom_settings.env
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Change this to a specific string.
+# Do not use "_" or other sepcial characters, only number and alphabeta.
+# eg: CONTAINER_UID="doris-jack-"
+# NOTICE: change this uid will modify the file in docker-compose.
+CONTAINER_UID="doris--"
diff --git a/docker/thirdparties/docker-compose/elasticsearch/es.yaml b/docker/thirdparties/docker-compose/elasticsearch/es.yaml.tpl
similarity index 100%
rename from docker/thirdparties/docker-compose/elasticsearch/es.yaml
rename to docker/thirdparties/docker-compose/elasticsearch/es.yaml.tpl
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl
similarity index 100%
rename from docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
rename to docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
similarity index 100%
rename from docker/thirdparties/docker-compose/hive/hive-2x.yaml
rename to docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
diff --git a/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl b/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
new file mode 100755
index 0000000000..e862778b63
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export SPARK_MASTER_HOST=doris--spark-iceberg
+
+start-master.sh -p 7077
+start-worker.sh spark://doris--spark-iceberg:7077
+start-history-server.sh
+start-thriftserver.sh
+
+# Entrypoint, for example notebook, pyspark or spark-sql
+if [[ $# -gt 0 ]]; then
+ eval "$1"
+fi
+
+# Avoid container exit
+while true; do
+ sleep 1
+done
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.env b/docker/thirdparties/docker-compose/iceberg/iceberg.env
new file mode 100644
index 0000000000..e4a95c99ce
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.env
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+NOTEBOOK_SERVER_PORT=8888
+SPARK_DRIVER_UI_PORT=8080
+SPARK_HISTORY_UI_PORT=10000
+REST_CATALOG_PORT=8181
+MINIO_UI_PORT=9000
+MINIO_API_PORT=9001
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
new file mode 100644
index 0000000000..fe8e29b77e
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+ doris--spark-iceberg:
+ image: tabulario/spark-iceberg
+ container_name: doris--spark-iceberg
+ hostname: doris--spark-iceberg
+ build: spark/
+ depends_on:
+ - doris--rest
+ - doris--minio
+ volumes:
+ - ./warehouse:/home/iceberg/warehouse
+ - ./notebooks:/home/iceberg/notebooks/notebooks
+ - ./entrypoint.sh:/opt/spark/entrypoint.sh
+ - ./spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
+ environment:
+ - AWS_ACCESS_KEY_ID=admin
+ - AWS_SECRET_ACCESS_KEY=password
+ - AWS_REGION=us-east-1
+ ports:
+ - ${NOTEBOOK_SERVER_PORT}:8888
+ - ${SPARK_DRIVER_UI_PORT}:8080
+ - ${SPARK_HISTORY_UI_PORT}:10000
+ links:
+ - doris--rest:rest
+ - doris--minio:minio
+ networks:
+ - doris--iceberg
+ entrypoint:
+ - /opt/spark/entrypoint.sh
+
+ doris--rest:
+ image: tabulario/iceberg-rest:0.2.0
+ ports:
+ - ${REST_CATALOG_PORT}:8181
+ environment:
+ - AWS_ACCESS_KEY_ID=admin
+ - AWS_SECRET_ACCESS_KEY=password
+ - AWS_REGION=us-east-1
+ - CATALOG_WAREHOUSE=s3a://warehouse/wh/
+ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+ - CATALOG_S3_ENDPOINT=http://doris--minio:9000
+ networks:
+ - doris--iceberg
+ doris--minio:
+ image: minio/minio
+ container_name: doris--minio
+ hostname: doris--minio
+ environment:
+ - MINIO_ROOT_USER=admin
+ - MINIO_ROOT_PASSWORD=password
+ ports:
+ - ${MINIO_UI_PORT}:9001
+ - ${MINIO_API_PORT}:9000
+ networks:
+ - doris--iceberg
+ command: ["server", "/data", "--console-address", ":9001"]
+ doris--mc:
+ depends_on:
+ - doris--minio
+ image: minio/mc
+ container_name: doris--mc
+ environment:
+ - AWS_ACCESS_KEY_ID=admin
+ - AWS_SECRET_ACCESS_KEY=password
+ - AWS_REGION=us-east-1
+ networks:
+ - doris--iceberg
+ entrypoint: >
+ /bin/sh -c "
+ until (/usr/bin/mc config host add minio http://doris--minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+ /usr/bin/mc rm -r --force minio/warehouse;
+ /usr/bin/mc mb minio/warehouse;
+ /usr/bin/mc policy set public minio/warehouse;
+ exit 0;
+ "
+networks:
+ doris--iceberg:
diff --git a/docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl b/docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl
new file mode 100644
index 0000000000..6e30b37164
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl
@@ -0,0 +1,11 @@
+spark.sql.catalog.demo org.apache.iceberg.spark.SparkCatalog
+spark.sql.catalog.demo.catalog-impl org.apache.iceberg.rest.RESTCatalog
+spark.sql.catalog.demo.uri http://doris--rest:8181
+spark.sql.catalog.demo.io-impl org.apache.iceberg.aws.s3.S3FileIO
+spark.sql.catalog.demo.warehouse s3a://warehouse/wh/
+spark.sql.catalog.demo.s3.endpoint http://doris--minio:9000
+spark.sql.defaultCatalog demo
+spark.eventLog.enabled true
+spark.eventLog.dir /home/iceberg/spark-events
+spark.history.fs.logDirectory /home/iceberg/spark-events
+spark.sql.catalogImplementation in-memory
diff --git a/docker/thirdparties/docker-compose/mysql/mysql-5.7.yaml b/docker/thirdparties/docker-compose/mysql/mysql-5.7.yaml.tpl
similarity index 100%
rename from docker/thirdparties/docker-compose/mysql/mysql-5.7.yaml
rename to docker/thirdparties/docker-compose/mysql/mysql-5.7.yaml.tpl
diff --git a/docker/thirdparties/docker-compose/oracle/oracle-11.yaml b/docker/thirdparties/docker-compose/oracle/oracle-11.yaml.tpl
similarity index 98%
rename from docker/thirdparties/docker-compose/oracle/oracle-11.yaml
rename to docker/thirdparties/docker-compose/oracle/oracle-11.yaml.tpl
index 93225aacd3..448b009170 100644
--- a/docker/thirdparties/docker-compose/oracle/oracle-11.yaml
+++ b/docker/thirdparties/docker-compose/oracle/oracle-11.yaml.tpl
@@ -46,4 +46,4 @@ services:
condition: service_healthy
networks:
- doris--oracle_11:
\ No newline at end of file
+ doris--oracle_11:
diff --git a/docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml b/docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl
similarity index 100%
rename from docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml
rename to docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl
diff --git a/docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml b/docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml.tpl
similarity index 100%
rename from docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml
rename to docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml.tpl
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
new file mode 100755
index 0000000000..bb68a2fe37
--- /dev/null
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -0,0 +1,251 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+################################################################
+# This script will restart all thirdparty containers
+################################################################
+
+set -eo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+. "${ROOT}/custom_settings.env"
+
+usage() {
+ echo "
+Usage: $0 <options>
+ Optional options:
+ [no option] start all components
+ --help,-h show this usage
+ -c mysql start MySQL
+ -c mysql,hive start MySQL and Hive
+ --stop stop the specified components
+
+ All valid components:
+ mysql,pg,oracle,sqlserver,es,hive,iceberg
+ "
+ exit 1
+}
+
+if ! OPTS="$(getopt \
+ -n "$0" \
+ -o '' \
+ -l 'help' \
+ -l 'stop' \
+ -o 'hc:' \
+ -- "$@")"; then
+ usage
+fi
+
+eval set -- "${OPTS}"
+
+COMPONENTS=""
+HELP=0
+STOP=0
+
+if [[ "$#" == 1 ]]; then
+ # default
+ COMPONENTS="mysql,pg,oracle,sqlserver,hive,iceberg"
+else
+ while true; do
+ case "$1" in
+ -h)
+ HELP=1
+ shift
+ ;;
+ --help)
+ HELP=1
+ shift
+ ;;
+ --stop)
+ STOP=1
+ shift
+ ;;
+ -c)
+ COMPONENTS=$2
+ shift 2
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ echo "Internal error"
+ exit 1
+ ;;
+ esac
+ done
+ if [[ "${COMPONENTS}"x == ""x ]]; then
+ if [[ "${STOP}" -eq 1 ]]; then
+ COMPONENTS="mysql,pg,oracle,sqlserver,hive,iceberg"
+ fi
+ fi
+fi
+
+if [[ "${HELP}" -eq 1 ]]; then
+ usage
+ exit 0
+fi
+
+if [[ "${COMPONENTS}"x == ""x ]]; then
+ echo "Invalid arguments"
+ usage
+ exit 1
+fi
+
+if [[ "${CONTAINER_UID}"x == "doris--"x ]]; then
+ echo "Must set CONTAINER_UID to a unique name in custom_settings.sh"
+ exit 1
+fi
+
+echo "Components are: ${COMPONENTS}"
+echo "Container UID: ${CONTAINER_UID}"
+echo "Stop: ${STOP}"
+
+OLD_IFS="${IFS}"
+IFS=','
+read -r -a COMPONENTS_ARR <<<"${COMPONENTS}"
+IFS="${OLD_IFS}"
+
+RUN_MYSQL=0
+RUN_PG=0
+RUN_ORACLE=0
+RUN_SQLSERVER=0
+RUN_HIVE=0
+RUN_ES=0
+RUN_ICEBERG=0
+for element in "${COMPONENTS_ARR[@]}"; do
+ if [[ "${element}"x == "mysql"x ]]; then
+ RUN_MYSQL=1
+ elif [[ "${element}"x == "pg"x ]]; then
+ RUN_PG=1
+ elif [[ "${element}"x == "oracle"x ]]; then
+ RUN_ORACLE=1
+ elif [[ "${element}"x == "sqlserver"x ]]; then
+ RUN_SQLSERVER=1
+ elif [[ "${element}"x == "es"x ]]; then
+ RUN_ES=1
+ elif [[ "${element}"x == "hive"x ]]; then
+ RUN_HIVE=1
+ elif [[ "${element}"x == "iceberg"x ]]; then
+ RUN_ICEBERG=1
+ else
+ echo "Invalid component: ${element}"
+ usage
+ exit 1
+ fi
+done
+
+if [[ "${RUN_ES}" -eq 1 ]]; then
+ # elasticsearch
+ cp "${ROOT}"/docker-compose/elasticsearch/es.yaml.tpl "${ROOT}"/docker-compose/elasticsearch/es.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/elasticsearch/es.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/elasticsearch/es.yaml --env-file "${ROOT}"/docker-compose/elasticsearch/es.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo mkdir -p "${ROOT}"/docker-compose/elasticsearch/data/es6/
+ sudo rm -rf "${ROOT}"/docker-compose/elasticsearch/data/es6/*
+ sudo mkdir -p "${ROOT}"/docker-compose/elasticsearch/data/es7/
+ sudo rm -rf "${ROOT}"/docker-compose/elasticsearch/data/es7/*
+ sudo mkdir -p "${ROOT}"/docker-compose/elasticsearch/data/es8/
+ sudo rm -rf "${ROOT}"/docker-compose/elasticsearch/data/es8/*
+ sudo chmod -R 777 "${ROOT}"/docker-compose/elasticsearch/data
+ sudo docker compose -f "${ROOT}"/docker-compose/elasticsearch/es.yaml --env-file "${ROOT}"/docker-compose/elasticsearch/es.env up -d --remove-orphans
+ fi
+fi
+
+if [[ "${RUN_MYSQL}" -eq 1 ]]; then
+ # mysql 5.7
+ cp "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml.tpl "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml --env-file "${ROOT}"/docker-compose/mysql/mysql-5.7.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo mkdir -p "${ROOT}"/docker-compose/mysql/data/
+ sudo rm "${ROOT}"/docker-compose/mysql/data/* -rf
+ sudo docker compose -f "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml --env-file "${ROOT}"/docker-compose/mysql/mysql-5.7.env up -d
+ fi
+fi
+
+if [[ "${RUN_PG}" -eq 1 ]]; then
+ # pg 14
+ cp "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml.tpl "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml --env-file "${ROOT}"/docker-compose/postgresql/postgresql-14.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo mkdir -p "${ROOT}"/docker-compose/postgresql/data/data
+ sudo rm "${ROOT}"/docker-compose/postgresql/data/data/* -rf
+ sudo docker compose -f "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml --env-file "${ROOT}"/docker-compose/postgresql/postgresql-14.env up -d
+ fi
+fi
+
+if [[ "${RUN_ORACLE}" -eq 1 ]]; then
+ # oracle
+ cp "${ROOT}"/docker-compose/oracle/oracle-11.yaml.tpl "${ROOT}"/docker-compose/oracle/oracle-11.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/oracle/oracle-11.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/oracle/oracle-11.yaml --env-file "${ROOT}"/docker-compose/oracle/oracle-11.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo mkdir -p "${ROOT}"/docker-compose/oracle/data/
+ sudo rm "${ROOT}"/docker-compose/oracle/data/* -rf
+ sudo docker compose -f "${ROOT}"/docker-compose/oracle/oracle-11.yaml --env-file "${ROOT}"/docker-compose/oracle/oracle-11.env up -d
+ fi
+fi
+
+if [[ "${RUN_SQLSERVER}" -eq 1 ]]; then
+ # sqlserver
+ cp "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml.tpl "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml --env-file "${ROOT}"/docker-compose/sqlserver/sqlserver.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo mkdir -p "${ROOT}"/docker-compose/sqlserver/data/
+ sudo rm "${ROOT}"/docker-compose/sqlserver/data/* -rf
+ sudo docker compose -f "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml --env-file "${ROOT}"/docker-compose/sqlserver/sqlserver.env up -d
+ fi
+fi
+
+if [[ "${RUN_HIVE}" -eq 1 ]]; then
+ # hive
+ # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
+ cp "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl "${ROOT}"/docker-compose/hive/hive-2x.yaml
+ cp "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl.tpl "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
+ sudo "${ROOT}"/docker-compose/hive/gen_env.sh
+ sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up -d
+ fi
+fi
+
+if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
+ # iceberg
+ cp "${ROOT}"/docker-compose/iceberg/iceberg.yaml.tpl "${ROOT}"/docker-compose/iceberg/iceberg.yaml
+ cp "${ROOT}"/docker-compose/iceberg/entrypoint.sh.tpl "${ROOT}"/docker-compose/iceberg/entrypoint.sh
+ cp "${ROOT}"/docker-compose/iceberg/spark-defaults.conf.tpl "${ROOT}"/docker-compose/iceberg/spark-defaults.conf
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/iceberg/iceberg.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/iceberg/entrypoint.sh
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/iceberg/spark-defaults.conf
+ sudo docker compose -f "${ROOT}"/docker-compose/iceberg/iceberg.yaml --env-file "${ROOT}"/docker-compose/iceberg/iceberg.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo rm -rf "${ROOT}"/docker-compose/iceberg/notebooks
+ sudo mkdir "${ROOT}"/docker-compose/iceberg/notebooks
+ sudo rm -rf "${ROOT}"/docker-compose/iceberg/spark
+ sudo mkdir "${ROOT}"/docker-compose/iceberg/spark
+ sudo rm -rf "${ROOT}"/docker-compose/iceberg/warehouse
+ sudo mkdir "${ROOT}"/docker-compose/iceberg/warehouse
+ sudo docker compose -f "${ROOT}"/docker-compose/iceberg/iceberg.yaml --env-file "${ROOT}"/docker-compose/iceberg/iceberg.env up -d
+ fi
+fi
diff --git a/docker/thirdparties/start-thirdparties-docker.sh b/docker/thirdparties/start-thirdparties-docker.sh
deleted file mode 100755
index e25be54711..0000000000
--- a/docker/thirdparties/start-thirdparties-docker.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-################################################################
-# This script will restart all thirdparty containers
-################################################################
-
-set -eo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
-
-# If you want to start multi group of these containers on same host,
-# Change this to a specific string.
-# Do not use "_" or other sepcial characters, only number and alphabeta.
-# NOTICE: change this uid will modify the file in docker-compose.
-CONTAINER_UID="doris--"
-
-# elasticsearch
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/elasticsearch/es.yaml
-sudo docker compose -f "${ROOT}"/docker-compose/elasticsearch/es.yaml --env-file "${ROOT}"/docker-compose/elasticsearch/es.env down
-sudo mkdir -p "${ROOT}"/docker-compose/elasticsearch/data/es6/
-sudo rm -rf "${ROOT}"/docker-compose/elasticsearch/data/es6/*
-sudo mkdir -p "${ROOT}"/docker-compose/elasticsearch/data/es7/
-sudo rm -rf "${ROOT}"/docker-compose/elasticsearch/data/es7/*
-sudo mkdir -p "${ROOT}"/docker-compose/elasticsearch/data/es8/
-sudo rm -rf "${ROOT}"/docker-compose/elasticsearch/data/es8/*
-sudo chmod -R 777 "${ROOT}"/docker-compose/elasticsearch/data
-sudo docker compose -f "${ROOT}"/docker-compose/elasticsearch/es.yaml --env-file "${ROOT}"/docker-compose/elasticsearch/es.env up -d --remove-orphans
-
-# mysql 5.7
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml
-sudo docker compose -f "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml --env-file "${ROOT}"/docker-compose/mysql/mysql-5.7.env down
-sudo mkdir -p "${ROOT}"/docker-compose/mysql/data/
-sudo rm "${ROOT}"/docker-compose/mysql/data/* -rf
-sudo docker compose -f "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml --env-file "${ROOT}"/docker-compose/mysql/mysql-5.7.env up -d
-
-# pg 14
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml
-sudo docker compose -f "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml --env-file "${ROOT}"/docker-compose/postgresql/postgresql-14.env down
-sudo mkdir -p "${ROOT}"/docker-compose/postgresql/data/data
-sudo rm "${ROOT}"/docker-compose/postgresql/data/data/* -rf
-sudo docker compose -f "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml --env-file "${ROOT}"/docker-compose/postgresql/postgresql-14.env up -d
-
-# oracle
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/oracle/oracle-11.yaml
-sudo docker compose -f "${ROOT}"/docker-compose/oracle/oracle-11.yaml --env-file "${ROOT}"/docker-compose/oracle/oracle-11.env down
-sudo mkdir -p "${ROOT}"/docker-compose/oracle/data/
-sudo rm "${ROOT}"/docker-compose/oracle/data/* -rf
-sudo docker compose -f "${ROOT}"/docker-compose/oracle/oracle-11.yaml --env-file "${ROOT}"/docker-compose/oracle/oracle-11.env up -d
-
-# sqlserver
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml
-sudo docker compose -f "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml --env-file "${ROOT}"/docker-compose/sqlserver/sqlserver.env down
-sudo mkdir -p "${ROOT}"/docker-compose/sqlserver/data/
-sudo rm "${ROOT}"/docker-compose/sqlserver/data/* -rf
-sudo docker compose -f "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml --env-file "${ROOT}"/docker-compose/sqlserver/sqlserver.env up -d
-
-# hive
-# before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml
-sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-sudo "${ROOT}"/docker-compose/hive/gen_env.sh
-sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
-sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up -d
diff --git a/docker/thirdparties/stop-thirdparties-docker.sh b/docker/thirdparties/stop-thirdparties-docker.sh
deleted file mode 100755
index 963bd1e85d..0000000000
--- a/docker/thirdparties/stop-thirdparties-docker.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-################################################################
-# This script will stop all thirdparty containers
-################################################################
-
-set -eo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
-
-# elasticsearch
-sudo docker compose -f "${ROOT}"/docker-compose/elasticsearch/es.yaml --env-file "${ROOT}"/docker-compose/elasticsearch/es.env down
-
-# mysql 5.7
-sudo docker compose -f "${ROOT}"/docker-compose/mysql/mysql-5.7.yaml --env-file "${ROOT}"/docker-compose/mysql/mysql-5.7.env down
-
-# pg 14
-sudo docker compose -f "${ROOT}"/docker-compose/postgresql/postgresql-14.yaml --env-file "${ROOT}"/docker-compose/postgresql/postgresql-14.env down
-
-# oracle 11
-sudo docker compose -f "${ROOT}"/docker-compose/oracle/oracle-11.yaml --env-file "${ROOT}"/docker-compose/oracle/oracle-11.env down
-
-# sqlserver
-sudo docker compose -f "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml --env-file "${ROOT}"/docker-compose/sqlserver/sqlserver.env down
-
-# hive
-sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
diff --git a/docs/zh-CN/community/developer-guide/regression-testing.md b/docs/zh-CN/community/developer-guide/regression-testing.md
index 53eba39099..60469ddbb3 100644
--- a/docs/zh-CN/community/developer-guide/regression-testing.md
+++ b/docs/zh-CN/community/developer-guide/regression-testing.md
@@ -595,15 +595,23 @@ JAVA_OPTS="-Dteamcity.enableStdErr=${enableStdErr}" ./run-regression-test.sh --t
Doris 支持一些外部署数据源的查询。所以回归框架也提供了通过 Docker Compose 搭建外部数据源的功能，以提供 Doris 对外部数据源的 e2e 测试。
+0. 准备工作
+
+ 在启动 Docker 前，请先修改 `docker/thirdparties/custom_settings.env` 文件中的 `CONTAINER_UID` 变量。
+
+ 可以修改为如：`doris-10002-18sda1-`。
+
+ 之后的启动脚本会，将 docker compose 中对应的名称进行替换，这样可以保证多套 containers 环境的容器名称和网络不会冲突。
+
1. 启动 Container
- Doris 目前支持 es, mysql, pg, hive 等数据源的 Docker compose。相关文件存放在 `docker/thirdparties/docker-compose` 目录下。
+ Doris 目前支持 es, mysql, pg, hive, sqlserver, oracle, iceberg 等数据源的 Docker compose。相关文件存放在 `docker/thirdparties/docker-compose` 目录下。
默认情况下，可以直接通过以下命令启动所有外部数据源的 Docker container：
（注意，hive container 需要下载预制的数据文件，请参阅下面 hive 相关的文档。）
```
- cd docker/thirdparties && sh start-thirdparties-docker.sh
+ cd docker/thirdparties && sh run-thirdparties-docker.sh
```
该命令需要 root 或 sudo 权限。命令返回成功，则代表所有 container 启动完成。可以通过 `docker ps -a` 命令查看。
@@ -611,46 +619,114 @@ Doris 支持一些外部署数据源的查询。所以回归框架也提供了
可以通过以下命令停止所有 container：
```
- cd docker/thirdparties && sh stop-thirdparties-docker.sh
+ cd docker/thirdparties && sh run-thirdparties-docker.sh --stop
+ ```
+
+ 也可以通过以下命令启动或停止指定的组件：
+
+ ```
+ cd docker/thirdparties
+ # 启动 mysql
+ sh run-thirdparties-docker.sh -c mysql
+ # 启动 mysql,pg,iceberg
+ sh run-thirdparties-docker.sh -c mysql,pg,iceberg
+ # 停止 mysql,pg,iceberg
+ sh run-thirdparties-docker.sh -c mysql,pg,iceberg --stop
```
1. MySQL
MySQL 相关的 Docker compose 文件存放在 docker/thirdparties/docker-compose/mysql 下。
- * `mysql-5.7.yaml`：Docker compose 文件，无需修改。默认用户名密码为 root/123456
+ * `mysql-5.7.yaml.tpl`：Docker compose 文件模板，无需修改。默认用户名密码为 root/123456
* `mysql-5.7.env`：配置文件，其中可以配置 MySQL container 对外暴露的端口，默认为 3316。
* `init/`：该目录存放的 sql 文件会在 container 创建后自动执行。目前默认会创建库、表并插入少量数据。
- * `data/`：container 启动后挂载的本地数据目录，`start-thirdparties-docker.sh` 脚本会在每次启动时，自动清空并重建这个目录。
+ * `data/`：container 启动后挂载的本地数据目录，`run-thirdparties-docker.sh` 脚本会在每次启动时，自动清空并重建这个目录。
2. Postgresql
Postgresql 相关的 Docker compose 文件存放在 docker/thirdparties/docker-compose/postgresql 下。
- * `postgresql-14.yaml`：Docker compose 文件，无需修改。默认用户名密码为 postgres/123456
+ * `postgresql-14.yaml.tpl`：Docker compose 文件模板，无需修改。默认用户名密码为 postgres/123456
* `postgresql-14.env`：配置文件，其中可以配置 Postgresql container 对外暴露的端口，默认为 5442。
* `init/`：该目录存放的 sql 文件会在 container 创建后自动执行。目前默认会创建库、表并插入少量数据。
- * `data/`：container 启动后挂载的本地数据目录，`start-thirdparties-docker.sh` 脚本会在每次启动时，自动清空并重建这个目录。
+ * `data/`：container 启动后挂载的本地数据目录，`run-thirdparties-docker.sh` 脚本会在每次启动时，自动清空并重建这个目录。
3. Hive
Hive 相关的 Docker compose 文件存放在 docker/thirdparties/docker-compose/hive 下。
- * `hive-2x.yaml`：Docker compose 文件，无需修改。
+ * `hive-2x.yaml.tpl`：Docker compose 文件模板，无需修改。
* `hadoop-hive.env.tpl`：配置文件的模板，无需修改。
- * `gen_env.sh`：初始化配置文件的脚本，可以在其中修改：`FS_PORT` 和 `HMS_PORT` 两个对外端口，分别对应 defaultFs 和 Hive metastore 的端口。默认为 8120 和 9183。`start-thirdparties-docker.sh` 启动时会自动调用这个脚本。
+ * `gen_env.sh`：初始化配置文件的脚本，可以在其中修改：`FS_PORT` 和 `HMS_PORT` 两个对外端口，分别对应 defaultFs 和 Hive metastore 的端口。默认为 8120 和 9183。`run-thirdparties-docker.sh` 启动时会自动调用这个脚本。
* `scripts/` 目录会在 container 启动后挂载到 container 中。其中的文件内容无需修改。但须注意，在启动 container 之前，需要先下载预制文件：
将 `https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/load/tpch1_parquet/tpch1.db.tar.gz` 文件下载到 `scripts/` 目录并解压即可。
4. Elasticsearch
- 包括 ES6，ES7，ES8 三个版本的 docker 镜像。docker/thirdparties/docker-compose/elasticsearch/ 下。
+ 包括 ES6，ES7，ES8 三个版本的 docker 镜像，存放在 docker/thirdparties/docker-compose/elasticsearch/ 下。
- * `es.yaml`：Docker compose文件。包括 ES6，ES7，ES8 三个版本。无需修改。
+ * `es.yaml.tpl`：Docker compose 文件模板。包括 ES6，ES7，ES8 三个版本。无需修改。
* `es.env`：配置文件，需配置 ES 的端口号。
* `scripts` 目录下存放了启动镜像后的初始化脚本。
+ 5. Oracle
+
+ 提供 Oracle 11 镜像，存放在 docker/thirdparties/docker-compose/oracle/ 下。
+
+ * `oracle-11.yaml.tpl`：Docker compose 文件模板。无需修改。
+ * `oracle-11.env`：配置 Oracle 对外端口，默认为 1521。
+
+ 6. SQLServer
+
+ 提供 SQLServer 2022 镜像，存放在 docker/thirdparties/docker-compose/sqlserver/ 下。
+
+ * `sqlserver.yaml.tpl`：Docker compose 文件模板。无需修改。
+ * `sqlserver.env`：配置 SQLServer 对外端口，默认为 1433。
+
+ 7. Iceberg
+
+ 提供 Iceberg + Spark + Minio 镜像组合。存放在 docker/thirdparties/docker-compose/iceberg/ 下。
+
+ * `iceberg.yaml.tpl`：Docker compose 文件模板。无需修改。
+ * `entrypoint.sh.tpl`：镜像启动后的初始化脚本模板。无需修改。
+ * `spark-defaults.conf.tpl`：Spark 的配置文件模板。无需修改。
+ * `iceberg.env`：对外端口配置文件，需修改各个对外端口，避免端口冲突。
+
+ 启动后，可以通过如下命令启动 spark-sql
+
+ `docker exec -it doris-xx-spark-iceberg spark-sql`
+
+ 其中 `doris-xx-spark-iceberg` 为 container 名称。
+
+ spark-sql iceberg 操作示例：
+
+ ```
+ create database db1;
+ show databases;
+ craete table db1.test1(k1 bigint, k2 bigint, k3 string) partition by (k1);
+ insert into db1.test1 values(1,2,'abc');
+ select * from db1.test1;
+ quit;
+ ```
+
+ 也可以通过 spark-shell 进行访问：
+
+ ```
+ docker exec -it doris-xx-spark-iceberg spark-shell
+
+ spark.sql(s"create database db1")
+ spark.sql(s"show databases").show()
+ spark.sql(s"craete table db1.test1(k1 bigint, k2 bigint, k3 string) partition by (k1)").show()
+ spark.sql(s"show tables from db1").show()
+ spark.sql(s"insert into db1.test1 values(1,2,'abc')").show()
+ spark.sql(s"select * from db1.test1").show()
+ :q
+ ```
+
+ 更多使用方式可参阅 [Tabular 官方文档](https://tabular.io/blog/docker-spark-and-iceberg/)。
+
2. 运行回归测试
外表相关的回归测试默认是关闭的，可以修改 `regression-test/conf/regression-conf.groovy` 中的以下配置来开启：
@@ -664,17 +740,4 @@ Doris 支持一些外部署数据源的查询。所以回归框架也提供了
* `es_7_port`：ES7 的端口。
* `es_8_port`：ES8 的端口。
-3. 如何在同一台机器上启动多套 Container。
-
- 如果需要在同一台机器上启动多套回归测试环境对应的 containers，除了修改上面所述的各种对外端口外，还需要修改 `start-thirdparties-docker.sh` 中的 `CONTAINER_UID` 配置。
-
- 默认为 `doris--`，可以修改为如：`doris-10002-18sda1-`。
-
- 修改后执行 `start-thirdparties-docker.sh`，脚本会将 docker compose 中对应的名称进行替换，这样可以保证多套 containers 环境的容器名称和网络不会冲突。
-
-
-
-
-
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 18/20: [enhencement](lock) print table lock owner when failed to try lock (#16186)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f1539758e3bdc614acc512c0d0a05436fe8c65c7
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Tue Jan 31 18:21:18 2023 +0800
[enhencement](lock) print table lock owner when failed to try lock (#16186)
---
.../main/java/org/apache/doris/catalog/Table.java | 22 ++++++++----
.../util/QueryableReentrantReadWriteLock.java | 41 ++++++++++++++++++++++
2 files changed, 57 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index f3c306a88c..5c8454fa0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.QueryableReentrantReadWriteLock;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.external.hudi.HudiTable;
@@ -49,7 +50,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
/**
@@ -70,7 +70,7 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
protected volatile String qualifiedDbName;
protected TableType type;
protected long createTime;
- protected ReentrantReadWriteLock rwLock;
+ protected QueryableReentrantReadWriteLock rwLock;
/*
* fullSchema and nameToColumn should contains all columns, both visible and shadow.
@@ -110,7 +110,7 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
this.type = type;
this.fullSchema = Lists.newArrayList();
this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
- this.rwLock = new ReentrantReadWriteLock(true);
+ this.rwLock = new QueryableReentrantReadWriteLock(true);
}
public Table(long id, String tableName, TableType type, List<Column> fullSchema) {
@@ -130,7 +130,7 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
// Only view in with-clause have null base
Preconditions.checkArgument(type == TableType.VIEW, "Table has no columns");
}
- this.rwLock = new ReentrantReadWriteLock();
+ this.rwLock = new QueryableReentrantReadWriteLock(true);
this.createTime = Instant.now().getEpochSecond();
}
@@ -148,7 +148,12 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
public boolean tryReadLock(long timeout, TimeUnit unit) {
try {
- return this.rwLock.readLock().tryLock(timeout, unit);
+ boolean res = this.rwLock.readLock().tryLock(timeout, unit);
+ if (!res && unit.toSeconds(timeout) >= 1) {
+ LOG.warn("Failed to try table {}'s read lock. timeout {} {}. Current owner: {}",
+ name, timeout, unit.name(), rwLock.getOwner());
+ }
+ return res;
} catch (InterruptedException e) {
LOG.warn("failed to try read lock at table[" + name + "]", e);
return false;
@@ -174,7 +179,12 @@ public abstract class Table extends MetaObject implements Writable, TableIf {
public boolean tryWriteLock(long timeout, TimeUnit unit) {
try {
- return this.rwLock.writeLock().tryLock(timeout, unit);
+ boolean res = this.rwLock.writeLock().tryLock(timeout, unit);
+ if (!res && unit.toSeconds(timeout) >= 1) {
+ LOG.warn("Failed to try table {}'s write lock. timeout {} {}. Current owner: {}",
+ name, timeout, unit.name(), rwLock.getOwner());
+ }
+ return res;
} catch (InterruptedException e) {
LOG.warn("failed to try write lock at table[" + name + "]", e);
return false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java
new file mode 100644
index 0000000000..3f55b54229
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.util;
+
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/*
+ * This Lock is for exposing the getOwner() method,
+ * which is a protected method of ReentrantLock
+ */
+public class QueryableReentrantReadWriteLock extends ReentrantReadWriteLock {
+ private static final long serialVersionUID = 1L;
+
+ public QueryableReentrantReadWriteLock() {
+ super();
+ }
+
+ public QueryableReentrantReadWriteLock(boolean fair) {
+ super(fair);
+ }
+
+ @Override
+ public Thread getOwner() {
+ return super.getOwner();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 11/20: [fix][FE] fix be coredump when children of FunctionCallExpr is folded (#16064)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9be0b42569d5176ae2c7be66ae16eca2c9b0f0db
Author: shee <13...@users.noreply.github.com>
AuthorDate: Mon Jan 30 15:25:00 2023 +0800
[fix][FE] fix be coredump when children of FunctionCallExpr is folded (#16064)
Co-authored-by: shizhiqiang03 <sh...@meituan.com>
fix be coredump when children of FunctionCallExpr is folded
---
.../mvrewrite/CountDistinctToBitmapOrHLLRule.java | 2 +-
.../test_count_distinct_with_case_function.out | 3 ++
.../test_count_distinct_with_case_function.groovy | 58 ++++++++++++++++++++++
3 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java
index e9e4795c31..b27169b448 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java
@@ -58,7 +58,7 @@ public class CountDistinctToBitmapOrHLLRule implements ExprRewriteRule {
return expr;
}
// rewrite expr
- FunctionParams newParams = new FunctionParams(false, fnExpr.getParams().exprs());
+ FunctionParams newParams = new FunctionParams(false, fnExpr.getChildren());
if (fnExpr.getChild(0).getType().isBitmapType()) {
FunctionCallExpr bitmapExpr = new FunctionCallExpr(FunctionSet.BITMAP_UNION_COUNT, newParams);
bitmapExpr.analyzeNoThrow(analyzer);
diff --git a/regression-test/data/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.out b/regression-test/data/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.out
new file mode 100644
index 0000000000..b3a2b05813
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+2
diff --git a/regression-test/suites/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.groovy b/regression-test/suites/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.groovy
new file mode 100644
index 0000000000..87c8369ab8
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/case_function/test_count_distinct_with_case_function.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_count_distinct_with_case_function") {
+ sql "DROP DATABASE IF EXISTS test_count_distinct_with_case_function"
+
+ sql "CREATE DATABASE test_count_distinct_with_case_function"
+
+ sql "USE test_count_distinct_with_case_function"
+
+ sql """
+ CREATE TABLE `a` (
+ `k1` int(11) NULL COMMENT "",
+ `k2` bitmap BITMAP_UNION NULL COMMENT "",
+ `k3` bitmap BITMAP_UNION NULL COMMENT ""
+ ) ENGINE=OLAP
+ AGGREGATE KEY(`k1`)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+ PROPERTIES (
+ "replication_num" = "1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql """
+ CREATE TABLE `b` (
+ `k1` int(11) NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+ PROPERTIES (
+ "replication_num" = "1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+ sql "insert into a values(1,to_bitmap(1),to_bitmap(1));"
+ sql "insert into a values(1,to_bitmap(1),to_bitmap(2));"
+ sql "insert into a values(2,to_bitmap(1),to_bitmap(1));"
+ sql "insert into b values(1);"
+
+ qt_select "select count(distinct case when false then k2 when true then k3 end) as tmp from a where k1 in (select k1 from b group by k1);"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 16/20: [Bug](function) fix now(int) use_default_implementation_for_nulls && fix dround signature (#16238)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 92df75c8c8385d7329f3b4794ec543152febd8eb
Author: Pxl <px...@qq.com>
AuthorDate: Mon Jan 30 18:01:26 2023 +0800
[Bug](function) fix now(int) use_default_implementation_for_nulls && fix dround signature (#16238)
---
be/src/vec/functions/function_date_or_datetime_computation.cpp | 2 +-
be/src/vec/functions/function_date_or_datetime_computation.h | 8 ++++++--
be/src/vec/functions/math.cpp | 1 +
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.cpp b/be/src/vec/functions/function_date_or_datetime_computation.cpp
index abaef68c4c..8df40c1fc3 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.cpp
+++ b/be/src/vec/functions/function_date_or_datetime_computation.cpp
@@ -89,7 +89,7 @@ using FunctionLocalTimestamp =
FunctionCurrentDateOrDateTime<CurrentDateTimeImpl<LocalTimestampFunctionName, false>>;
using FunctionNowWithPrecision =
- FunctionCurrentDateOrDateTime<CurrentDateTimeImpl<NowFunctionName, true>>;
+ FunctionCurrentDateOrDateTime<CurrentDateTimeImpl<NowFunctionName, true>, false>;
using FunctionCurrentTimestampWithPrecision =
FunctionCurrentDateOrDateTime<CurrentDateTimeImpl<CurrentTimestampFunctionName, true>>;
using FunctionLocalTimeWithPrecision =
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h
index eba6d17fc7..a4d51c972c 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.h
+++ b/be/src/vec/functions/function_date_or_datetime_computation.h
@@ -583,7 +583,7 @@ public:
}
};
-template <typename FunctionImpl>
+template <typename FunctionImpl, bool DefaultNullable = true>
class FunctionCurrentDateOrDateTime : public IFunction {
public:
static constexpr bool has_variadic_argument =
@@ -596,6 +596,8 @@ public:
size_t get_number_of_arguments() const override { return 0; }
+ bool use_default_implementation_for_nulls() const override { return DefaultNullable; }
+
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
return std::make_shared<typename FunctionImpl::ReturnType>();
}
@@ -603,7 +605,9 @@ public:
bool is_variadic() const override { return true; }
DataTypes get_variadic_argument_types_impl() const override {
- if constexpr (has_variadic_argument) return FunctionImpl::get_variadic_argument_types();
+ if constexpr (has_variadic_argument) {
+ return FunctionImpl::get_variadic_argument_types();
+ }
return {};
}
diff --git a/be/src/vec/functions/math.cpp b/be/src/vec/functions/math.cpp
index 67552b0f18..504684ccf7 100644
--- a/be/src/vec/functions/math.cpp
+++ b/be/src/vec/functions/math.cpp
@@ -425,6 +425,7 @@ void register_function_math(SimpleFunctionFactory& factory) {
REGISTER_ROUND_FUNCTIONS(DecimalRoundTwoImpl)
REGISTER_ROUND_FUNCTIONS(DoubleRoundOneImpl)
REGISTER_ROUND_FUNCTIONS(DoubleRoundTwoImpl)
+ factory.register_alias("round", "dround");
factory.register_function<FunctionAcos>();
factory.register_function<FunctionAsin>();
factory.register_function<FunctionAtan>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/20: [enhancement](FE)shut down fast throw JVM might do when throwing exception #16146
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fb7944cd936c55108c8105f24549a936f0b72c0b
Author: AlexYue <yj...@gmail.com>
AuthorDate: Sat Jan 28 14:18:25 2023 +0800
[enhancement](FE)shut down fast throw JVM might do when throwing exception #16146
As discussed in 16107
Sometimes jvm would try to reduce the whole stack to just one line, it's kind of confusing for debugging.
Issue Number: close #xxx
---
bin/start_fe.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bin/start_fe.sh b/bin/start_fe.sh
index fc7ec47e39..5028baedaf 100755
--- a/bin/start_fe.sh
+++ b/bin/start_fe.sh
@@ -210,10 +210,10 @@ if [[ "${IMAGE_TOOL}" -eq 1 ]]; then
echo "Internal Error. USE IMAGE_TOOL like : ./start_fe.sh --image image_path"
fi
elif [[ "${RUN_DAEMON}" -eq 1 ]]; then
- nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER:+${HELPER}} "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null &
+ nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER:+${HELPER}} "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null &
else
export DORIS_LOG_TO_STDERR=1
- ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "$@" </dev/null
+ ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "$@" </dev/null
fi
echo $! >"${pidfile}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/20: [Fix](profile) do not send export profile when enable_profile=false. (#15996)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 24f2511a39bff7ca8e6fc0cfc39f8865b1723c5c
Author: wxy <du...@gmail.com>
AuthorDate: Thu Jan 19 08:06:39 2023 +0800
[Fix](profile) do not send export profile when enable_profile=false. (#15996)
---
.../main/java/org/apache/doris/load/ExportJob.java | 10 +-
.../org/apache/doris/task/ExportExportingTask.java | 3 +
.../org/apache/doris/qe/SessionVariablesTest.java | 131 ++++++++++++++++++---
3 files changed, 125 insertions(+), 19 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index a7f6bef921..3b79ca70de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -129,12 +129,11 @@ public class ExportJob implements Writable {
private String lineDelimiter;
private Map<String, String> properties = Maps.newHashMap();
private List<String> partitions;
-
private TableName tableName;
-
private String sql = "";
-
private JobState state;
+ // If set to true, the profile of export job with be pushed to ProfileManager
+ private volatile boolean enableProfile = false;
private long createTimeMs;
private long startTimeMs;
private long finishTimeMs;
@@ -239,6 +238,7 @@ public class ExportJob implements Writable {
if (ConnectContext.get() != null) {
SessionVariable var = ConnectContext.get().getSessionVariable();
this.sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
+ this.enableProfile = var.enableProfile();
} else {
this.sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
}
@@ -751,6 +751,10 @@ public class ExportJob implements Writable {
return queryId;
}
+ public boolean getEnableProfile() {
+ return enableProfile;
+ }
+
@Override
public String toString() {
return "ExportJob [jobId=" + id
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
index fe6dde27fb..7d6a3b3c74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
@@ -271,6 +271,9 @@ public class ExportExportingTask extends MasterTask {
}
private void registerProfile() {
+ if (!job.getEnableProfile()) {
+ return;
+ }
initProfile();
for (RuntimeProfile p : fragmentProfiles) {
profile.addChild(p);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
index c1911cc687..947e83d95d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
@@ -17,24 +17,39 @@
package org.apache.doris.qe;
+import org.apache.doris.analysis.ExportStmt;
+import org.apache.doris.analysis.SetStmt;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.util.ProfileManager;
+import org.apache.doris.common.util.RuntimeProfile;
+import org.apache.doris.load.ExportJob;
+import org.apache.doris.task.ExportExportingTask;
import org.apache.doris.thrift.TQueryOptions;
+import org.apache.doris.utframe.TestWithFeService;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import com.google.common.collect.Lists;
+import mockit.Expectations;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.util.Map;
-public class SessionVariablesTest {
+public class SessionVariablesTest extends TestWithFeService {
- private static SessionVariable sessionVariable;
- private static int numOfForwardVars;
+ private SessionVariable sessionVariable;
+ private int numOfForwardVars;
+ private ProfileManager profileManager = ProfileManager.getInstance();
- @BeforeClass
- public static void beforeClass() throws Exception {
- sessionVariable = new SessionVariable();
+ @Override
+ protected void runBeforeAll() throws Exception {
+ FeConstants.runningUnitTest = true;
+ createDatabase("test_d");
+ useDatabase("test_d");
+ createTable("create table test_t1 \n" + "(k1 int, k2 int) distributed by hash(k1) buckets 1\n"
+ + "properties(\"replication_num\" = \"1\");");
+ sessionVariable = new SessionVariable();
Field[] fields = SessionVariable.class.getFields();
for (Field f : fields) {
VariableMgr.VarAttr varAttr = f.getAnnotation(VariableMgr.VarAttr.class);
@@ -48,23 +63,107 @@ public class SessionVariablesTest {
@Test
public void testForwardSessionVariables() {
Map<String, String> vars = sessionVariable.getForwardVariables();
- Assert.assertTrue(numOfForwardVars >= 6);
- Assert.assertEquals(numOfForwardVars, vars.size());
+ Assertions.assertTrue(numOfForwardVars >= 6);
+ Assertions.assertEquals(numOfForwardVars, vars.size());
vars.put(SessionVariable.ENABLE_PROFILE, "true");
sessionVariable.setForwardedSessionVariables(vars);
- Assert.assertEquals(true, sessionVariable.enableProfile);
+ Assertions.assertEquals(true, sessionVariable.enableProfile);
}
@Test
public void testForwardQueryOptions() {
TQueryOptions queryOptions = sessionVariable.getQueryOptionVariables();
- Assert.assertTrue(queryOptions.isSetMemLimit());
- Assert.assertFalse(queryOptions.isSetLoadMemLimit());
- Assert.assertTrue(queryOptions.isSetQueryTimeout());
+ Assertions.assertTrue(queryOptions.isSetMemLimit());
+ Assertions.assertFalse(queryOptions.isSetLoadMemLimit());
+ Assertions.assertTrue(queryOptions.isSetQueryTimeout());
queryOptions.setQueryTimeout(123);
sessionVariable.setForwardedSessionVariables(queryOptions);
- Assert.assertEquals(123, sessionVariable.getQueryTimeoutS());
+ Assertions.assertEquals(123, sessionVariable.getQueryTimeoutS());
+ }
+
+ @Test
+ public void testEnableProfile() {
+ try {
+ SetStmt setStmt = (SetStmt) parseAndAnalyzeStmt("set enable_profile=true", connectContext);
+ SetExecutor setExecutor = new SetExecutor(connectContext, setStmt);
+ setExecutor.execute();
+
+ ExportStmt exportStmt = (ExportStmt)
+ parseAndAnalyzeStmt("EXPORT TABLE test_d.test_t1 TO \"file:///tmp/test_t1\"", connectContext);
+ ExportJob job = new ExportJob(1234);
+ job.setJob(exportStmt);
+
+ new Expectations(job) {
+ {
+ job.getState();
+ minTimes = 0;
+ result = ExportJob.JobState.EXPORTING;
+
+ job.getCoordList();
+ minTimes = 0;
+ result = Lists.newArrayList();
+ }
+ };
+
+ new Expectations(profileManager) {
+ {
+ profileManager.pushProfile((RuntimeProfile) any);
+ // if enable_profile=true, method pushProfile will be called once
+ times = 1;
+ }
+ };
+
+ ExportExportingTask task = new ExportExportingTask(job);
+ task.run();
+ Assertions.assertTrue(job.isFinalState());
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assertions.fail(e.getMessage());
+ }
+
+ }
+
+ @Test
+ public void testDisableProfile() {
+ try {
+ SetStmt setStmt = (SetStmt) parseAndAnalyzeStmt("set enable_profile=false", connectContext);
+ SetExecutor setExecutor = new SetExecutor(connectContext, setStmt);
+ setExecutor.execute();
+
+ ExportStmt exportStmt = (ExportStmt)
+ parseAndAnalyzeStmt("EXPORT TABLE test_d.test_t1 TO \"file:///tmp/test_t1\"", connectContext);
+ ExportJob job = new ExportJob(1234);
+ job.setJob(exportStmt);
+
+ new Expectations(job) {
+ {
+ job.getState();
+ minTimes = 0;
+ result = ExportJob.JobState.EXPORTING;
+
+ job.getCoordList();
+ minTimes = 0;
+ result = Lists.newArrayList();
+ }
+ };
+
+ new Expectations(profileManager) {
+ {
+ profileManager.pushProfile((RuntimeProfile) any);
+ // if enable_profile=false, method pushProfile will not be called
+ times = 0;
+ }
+ };
+
+ ExportExportingTask task = new ExportExportingTask(job);
+ task.run();
+ Assertions.assertTrue(job.isFinalState());
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assertions.fail(e.getMessage());
+ }
+
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 20/20: [improvement](agg-function) Increase the limit maximum number of agg function parameters (#15924)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d9666a694bea53c0dea1c2145fd08f9c0ed59799
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Tue Jan 31 21:03:50 2023 +0800
[improvement](agg-function) Increase the limit maximum number of agg function parameters (#15924)
---
.../aggregate_functions/aggregate_function_null.h | 4 ++-
be/test/vec/exprs/vexpr_test.cpp | 15 +++-----
.../test_aggregate_retention.out | 26 ++++++++++++++
.../test_aggregate_retention.sql | 42 +++++++++++++++++++++-
4 files changed, 75 insertions(+), 12 deletions(-)
diff --git a/be/src/vec/aggregate_functions/aggregate_function_null.h b/be/src/vec/aggregate_functions/aggregate_function_null.h
index 7b8489c095..fea150348f 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_null.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_null.h
@@ -315,7 +315,9 @@ public:
}
private:
- enum { MAX_ARGS = 8 };
+ // The array length is fixed in the implementation of some aggregate functions.
+ // Therefore we choose 256 as the appropriate maximum length limit.
+ static const size_t MAX_ARGS = 256;
size_t number_of_arguments = 0;
std::array<char, MAX_ARGS>
is_nullable; /// Plain array is better than std::vector due to one indirection less.
diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp
index ca8f8938a8..9ebf5612a8 100644
--- a/be/test/vec/exprs/vexpr_test.cpp
+++ b/be/test/vec/exprs/vexpr_test.cpp
@@ -79,7 +79,7 @@ TEST(TEST_VEXPR, ABSTEST) {
auto block = row_batch.convert_to_vec_block();
int ts = -1;
- context->execute(&block, &ts);
+ state = context->execute(&block, &ts);
context->close(&runtime_stat);
}
@@ -119,20 +119,15 @@ TEST(TEST_VEXPR, ABSTEST2) {
DescriptorTbl desc_tbl;
desc_tbl._slot_desc_map[0] = tuple_desc->slots()[0];
runtime_stat.set_desc_tbl(&desc_tbl);
-<<<<<<< HEAD
- context->prepare(&runtime_stat, row_desc);
- context->open(&runtime_stat);
-
- auto block = row_batch.convert_to_vec_block();
- int ts = -1;
- context->execute(&block, &ts);
-=======
auto state = Status::OK();
state = context->prepare(&runtime_stat, row_desc);
ASSERT_TRUE(state.ok());
state = context->open(&runtime_stat);
ASSERT_TRUE(state.ok());
->>>>>>> 46347a51d2... [Bug](exec) enable warning on ignoring function return value for vctx (#16157)
+
+ auto block = row_batch.convert_to_vec_block();
+ int ts = -1;
+ state = context->execute(&block, &ts);
context->close(&runtime_stat);
}
diff --git a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.out b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.out
index d5d1ef2bc5..a85e7c5e37 100644
--- a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.out
+++ b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.out
@@ -60,3 +60,29 @@
-- !test_aggregate_retention_13 --
3 2 1
+-- !test_aggregate_retention_14 --
+0
+
+-- !test_aggregate_retention_15 --
+0
+
+-- !test_aggregate_retention_16 --
+12
+
+-- !test_aggregate_retention_17 --
+0 2022-10-12T00:00
+0 2022-10-13T00:00
+0 2022-10-14T00:00
+0 2022-10-15T00:00
+0 2022-10-16T00:00
+0 2022-10-17T00:00
+0 2022-10-18T00:00
+0 2022-10-19T00:00
+0 2022-10-20T00:00
+0 2022-10-21T00:00
+0 2022-10-22T00:00
+0 2022-10-23T00:00
+
+-- !test_aggregate_retention_18 --
+0 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+
diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
index bdcfec400b..51fa7c677f 100644
--- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
+++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
@@ -85,4 +85,44 @@ SELECT
AS r
FROM retention_test
GROUP BY uid
- ) a;
\ No newline at end of file
+ ) a;
+
+
+DROP TABLE IF EXISTS retention_test_many_params;
+
+CREATE TABLE IF NOT EXISTS retention_test_many_params(
+ `uid` int COMMENT 'user id',
+ `date` datetime COMMENT 'date time'
+ )
+DUPLICATE KEY(uid)
+DISTRIBUTED BY HASH(uid) BUCKETS 1
+PROPERTIES (
+ "replication_num" = "1"
+);
+
+INSERT into retention_test_many_params (uid, date) values (0, '2022-10-12'),
+ (0, '2022-10-13'),
+ (0, '2022-10-14'),
+ (0, '2022-10-15'),
+ (0, '2022-10-16'),
+ (0, '2022-10-17'),
+ (0, '2022-10-18'),
+ (0, '2022-10-19'),
+ (0, '2022-10-20'),
+ (0, '2022-10-21'),
+ (0, '2022-10-22'),
+ (0, '2022-10-23');
+
+SELECT * from retention_test_many_params ORDER BY date;
+
+SELECT
+ uid,
+ retention(date = '2022-10-12', date = '2022-10-13', date = '2022-10-14',
+ date = '2022-10-15', date = '2022-10-16', date = '2022-10-17',
+ date = '2022-10-18', date = '2022-10-19', date = '2022-10-20',
+ date = '2022-10-21', date = '2022-10-22', date = '2022-10-23'
+ )
+ AS r
+ FROM retention_test_many_params
+ GROUP BY uid
+ ORDER BY uid ASC;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/20: [Feature](profile) set sql and defaultDb fields in show-load-profile. (#15875)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 749819c390e5ae3f0f0ac610ad623563f270167c
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Sat Jan 21 08:10:15 2023 +0800
[Feature](profile) set sql and defaultDb fields in show-load-profile. (#15875)
When execute show load profile '/', the value of SQL and DefaultDb columns are all 'N/A', but we can fill these fields，the result of this pr is as follows:
Execute show load profile '/'\G:
MySQL [test_d]> show load profile '/'\G
*************************** 1. row ***************************
QueryId: 652326
User: N/A
DefaultDb: default_cluster:test_d
SQL: LOAD LABEL `default_cluster:test_d`.`xxx` (APPEND DATA INFILE ('hdfs://xxx/user/hive/warehouse/xxx.db/xxx/*') INTO TABLE xxx FORMAT AS 'ORC' (c1, c2, c3) SET (`c1` = `c1`, `c2` = `c2`, `c3` = `c3`)) WITH BROKER broker_xxx (xxx) PROPERTIES ("max_filter_ratio" = "0", "timeout" = "30000")
QueryType: Load
StartTime: 2023-01-12 18:33:34
EndTime: 2023-01-12 18:33:46
TotalTime: 11s613ms
QueryState: N/A
1 row in set (0.01 sec)
---
.../main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 9 +++++++--
.../src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java | 4 ++++
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index 5d42fff639..31c07820cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -326,8 +326,8 @@ public class BrokerLoadJob extends BulkLoadJob {
summaryProfile.addInfoString(ProfileManager.QUERY_TYPE, "Load");
summaryProfile.addInfoString(ProfileManager.QUERY_STATE, "N/A");
summaryProfile.addInfoString(ProfileManager.USER, "N/A");
- summaryProfile.addInfoString(ProfileManager.DEFAULT_DB, "N/A");
- summaryProfile.addInfoString(ProfileManager.SQL_STATEMENT, "N/A");
+ summaryProfile.addInfoString(ProfileManager.DEFAULT_DB, getDefaultDb());
+ summaryProfile.addInfoString(ProfileManager.SQL_STATEMENT, this.getOriginStmt().originStmt);
summaryProfile.addInfoString(ProfileManager.IS_CACHED, "N/A");
// Add the summary profile to the first
@@ -336,6 +336,11 @@ public class BrokerLoadJob extends BulkLoadJob {
ProfileManager.getInstance().pushProfile(jobProfile);
}
+ private String getDefaultDb() {
+ Database database = Env.getCurrentEnv().getInternalCatalog().getDb(this.dbId).orElse(null);
+ return database == null ? "N/A" : database.getFullName();
+ }
+
private void updateLoadingStatus(BrokerLoadingTaskAttachment attachment) {
loadingStatus.replaceCounter(DPP_ABNORMAL_ALL,
increaseCounter(DPP_ABNORMAL_ALL, attachment.getCounter(DPP_ABNORMAL_ALL)));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
index a394a62d4a..602fd7a42d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
@@ -306,6 +306,10 @@ public abstract class BulkLoadJob extends LoadJob {
}
}
+ public OriginStatement getOriginStmt() {
+ return this.originStmt;
+ }
+
public void readFields(DataInput in) throws IOException {
super.readFields(in);
brokerDesc = BrokerDesc.read(in);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 09/20: [Improvement](decimal) do not log fatal when precision is invalid (#16207)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bf5417e80a2c874136cd688e77aa21f3d177dfc2
Author: Gabriel <ga...@gmail.com>
AuthorDate: Mon Jan 30 09:54:22 2023 +0800
[Improvement](decimal) do not log fatal when precision is invalid (#16207)
---
be/src/vec/data_types/data_type_decimal.cpp | 6 ++++--
be/src/vec/exprs/vexpr.cpp | 3 +++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/data_types/data_type_decimal.cpp b/be/src/vec/data_types/data_type_decimal.cpp
index aa1da12e4d..5d00e65033 100644
--- a/be/src/vec/data_types/data_type_decimal.cpp
+++ b/be/src/vec/data_types/data_type_decimal.cpp
@@ -163,11 +163,13 @@ T DataTypeDecimal<T>::parse_from_string(const std::string& str) const {
DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value, bool use_v2) {
if (precision_value < min_decimal_precision() ||
precision_value > max_decimal_precision<Decimal128>()) {
- LOG(FATAL) << "Wrong precision " << precision_value;
+ LOG(WARNING) << "Wrong precision " << precision_value;
+ return nullptr;
}
if (static_cast<UInt64>(scale_value) > precision_value) {
- LOG(FATAL) << "Negative scales and scales larger than precision are not supported";
+ LOG(WARNING) << "Negative scales and scales larger than precision are not supported";
+ return nullptr;
}
if (use_v2) {
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index b1dbe2015f..a91109291d 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -160,6 +160,9 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
+ if (!(*expr)->data_type()) {
+ return Status::InvalidArgument("Unknown expr type: {}", texpr_node.node_type);
+ }
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org