You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/05/13 08:10:43 UTC
[doris] branch branch-1.2-lts updated (a3edd41db8 -> 77b8463886)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from a3edd41db8 [branch1.2](fix) avoiding using broker reader when file size can not be got (#19474)
new 970995b98a [fix](metric) Fix bug for that register txn replica failed (#17855)
new 49e8550e9a [fix](planner) insubquery should always be converted to semi or anti join (#19240)
new fffd4e4bc9 [Enhancement](storage engine) avoid deleting tablets on unused disk (#19010)
new f187424885 [fix](memory) Fix memory leak due to incorrect block reuse of AggregateFunctionSortData #19214
new 1225094873 [enhancement](planner) fix unclear exception msg when create mv (#17537)
new 9be8263050 [fix](fe)havingClause should be substituted the same way as resultExprs (#19261)
new 9a51fc761e [fix](JDBC) set jdbc parameters to compatible with both MySQL and Doris when reading boolean type (#19399)
new de0139d5cc [fix](catalog) fix doris jdbc catalog largeint select error (#19407)
new c8f4873393 [fix](MySQL) the way Doris handles boolean type is consistent with MySQL (#19416)
new 7fd58d60c7 [Enhancement](multi-catalog) add config for external meta cache loade… (#18117)
new abc4483d00 [Bug](scan) forbiden push down in predicate when in_state->use_set is false (#19471)
new 80f1508065 [Fix](multi catalog)Return all partition values while reading hive table. (#19434)
new d9fe522cca [minor](log) change debug log to info to observe the storage medium change #19529
new 77b8463886 [branch1.2](fix) fix compilation error after cherry-pick
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/olap/storage_engine.cpp | 18 +++----------
be/src/olap/storage_engine.h | 2 +-
.../aggregate_functions/aggregate_function_sort.h | 2 +-
be/src/vec/exec/scan/vscan_node.cpp | 12 +++++++++
be/src/vec/exec/vjdbc_connector.cpp | 2 +-
.../main/java/org/apache/doris/common/Config.java | 13 ++++++++++
.../org/apache/doris/analysis/BinaryPredicate.java | 4 +++
.../doris/analysis/CreateMaterializedViewStmt.java | 4 +++
.../java/org/apache/doris/analysis/SelectStmt.java | 8 ++++--
.../org/apache/doris/analysis/StmtRewriter.java | 10 +++++---
.../org/apache/doris/analysis/StringLiteral.java | 9 ++++++-
.../org/apache/doris/catalog/DataProperty.java | 12 +++++++--
.../main/java/org/apache/doris/catalog/Env.java | 6 +++--
.../org/apache/doris/catalog/JdbcResource.java | 11 +++++---
.../org/apache/doris/catalog/PartitionInfo.java | 4 +--
.../doris/datasource/ExternalMetaCacheMgr.java | 4 ++-
.../doris/datasource/hive/HiveMetaStoreCache.java | 2 +-
.../datasource/hive/PooledHiveMetaStoreClient.java | 3 ++-
.../doris/load/routineload/KafkaTaskInfo.java | 9 -------
.../java/org/apache/doris/metric/MetricRepo.java | 13 ----------
.../org/apache/doris/planner/OlapTableSink.java | 11 --------
.../doris/transaction/DatabaseTransactionMgr.java | 30 +---------------------
.../doris/transaction/GlobalTransactionMgr.java | 9 ++-----
.../apache/doris/transaction/TransactionState.java | 8 ------
.../java/org/apache/doris/udf/JdbcExecutor.java | 16 ++++++++++++
.../correctness_p0/test_group_having_alias.out | 3 +++
.../data/query_p0/sql_functions/test_in_expr.out | 10 ++++++++
.../correctness_p0/test_group_having_alias.groovy | 3 +++
.../test_join_without_condition.groovy | 6 +++++
.../query_p0/sql_functions/test_in_expr.groovy | 11 ++++++++
30 files changed, 141 insertions(+), 114 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 10/14: [Enhancement](multi-catalog) add config for external meta cache loade… (#18117)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7fd58d60c7d00fbb38eba9c87fa7ff0a44b8ff42
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Tue Mar 28 15:10:19 2023 +0800
[Enhancement](multi-catalog) add config for external meta cache loade… (#18117)
Add config for external cache-loader's max thread-pool size.
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 7 +++++++
.../java/org/apache/doris/datasource/ExternalMetaCacheMgr.java | 4 +++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f40e7233cd..0466e31f1d 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1900,6 +1900,13 @@ public class Config extends ConfigBase {
@ConfField(mutable = false, masterOnly = false)
public static long max_hive_partition_cache_num = 100000;
+ /**
+ * Max cache loader thread-pool size.
+ * Max thread pool size for loading external meta cache
+ */
+ @ConfField(mutable = false, masterOnly = false)
+ public static int max_external_cache_loader_thread_pool_size = 10;
+
/**
* Max cache num of external catalog's file
* Decrease this value if FE's memory is small
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index 13875d084e..afe1cce61c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.cluster.ClusterNamespace;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
@@ -46,7 +47,8 @@ public class ExternalMetaCacheMgr {
private Executor executor;
public ExternalMetaCacheMgr() {
- executor = ThreadPoolManager.newDaemonCacheThreadPool(10, "ExternalMetaCacheMgr", false);
+ executor = ThreadPoolManager.newDaemonCacheThreadPool(Config.max_external_cache_loader_thread_pool_size,
+ "ExternalMetaCacheMgr", true);
}
public HiveMetaStoreCache getMetaStoreCache(HMSExternalCatalog catalog) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 09/14: [fix](MySQL) the way Doris handles boolean type is consistent with MySQL (#19416)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c8f487339387b80c6dedabb79e0782f1191aaaa0
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Wed May 10 00:58:09 2023 +0800
[fix](MySQL) the way Doris handles boolean type is consistent with MySQL (#19416)
---
.../src/main/java/org/apache/doris/analysis/BinaryPredicate.java | 4 ++++
.../src/main/java/org/apache/doris/analysis/StringLiteral.java | 9 ++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
index 2b01f17319..11504d9571 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
@@ -380,6 +380,10 @@ public class BinaryPredicate extends Predicate implements Writable {
&& (t2 == PrimitiveType.BIGINT || t2 == PrimitiveType.LARGEINT)) {
return Type.LARGEINT;
}
+ // MySQL will try to parse string as bigint, if failed, will take string as 0.
+ if (t1 == PrimitiveType.BIGINT && t2.isCharFamily()) {
+ return Type.BIGINT;
+ }
// Implicit conversion affects query performance.
// For a common example datekey='20200825' which datekey is int type.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
index 62038dec27..1804730663 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
@@ -226,7 +226,14 @@ public class StringLiteral extends LiteralExpr {
throw new AnalysisException(e.getMessage());
}
}
- return new IntLiteral(value, targetType);
+ // MySQL will try to parse string as bigint, if failed, will cast string as 0.
+ long longValue;
+ try {
+ longValue = Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ longValue = 0L;
+ }
+ return new IntLiteral(longValue, targetType);
case LARGEINT:
if (VariableVarConverters.hasConverter(beConverted)) {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/14: [fix](memory) Fix memory leak due to incorrect block reuse of AggregateFunctionSortData #19214
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f1874248858006e4dea8aefab359157e17fdc6be
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Fri May 5 14:29:34 2023 +0800
[fix](memory) Fix memory leak due to incorrect block reuse of AggregateFunctionSortData #19214
---
be/src/vec/aggregate_functions/aggregate_function_sort.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.h b/be/src/vec/aggregate_functions/aggregate_function_sort.h
index 1f19996f04..2fe23ebcea 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sort.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sort.h
@@ -66,7 +66,7 @@ struct AggregateFunctionSortData {
PBlock pblock;
pblock.ParseFromString(data);
- new (&block) Block(pblock);
+ block = Block(pblock);
}
void add(const IColumn** columns, size_t columns_num, size_t row_num) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 12/14: [Fix](multi catalog)Return all partition values while reading hive table. (#19434)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 80f1508065ae6de893a65058ad007e8274d2fba9
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed May 10 10:55:33 2023 +0800
[Fix](multi catalog)Return all partition values while reading hive table. (#19434)
Return all partition values while reading hive table.
Add a config item for the max value of hive table to partition list cache.
Default value is 100.
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 8 ++++++++
.../java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 2 +-
.../apache/doris/datasource/hive/PooledHiveMetaStoreClient.java | 3 ++-
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 0466e31f1d..9e3c3aac3e 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1900,6 +1900,14 @@ public class Config extends ConfigBase {
@ConfField(mutable = false, masterOnly = false)
public static long max_hive_partition_cache_num = 100000;
+ @ConfField(mutable = false, masterOnly = false, description = {"Hive表到分区名列表缓存的最大数量。",
+ "Max cache number of hive table to partition names list."})
+ public static long max_hive_table_catch_num = 1000;
+
+ @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量,-1代表没有限制。",
+ "Max number of hive partition values to return while list partitions, -1 means no limitation."})
+ public static short max_hive_list_partition_num = -1;
+
/**
* Max cache loader thread-pool size.
* Max thread pool size for loading external meta cache
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index a2fb14de39..95f0b957c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -99,7 +99,7 @@ public class HiveMetaStoreCache {
}
private void init(Executor executor) {
- partitionValuesCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_partition_cache_num)
+ partitionValuesCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_table_catch_num)
.expireAfterAccess(Config.external_cache_expire_time_minutes_after_access, TimeUnit.MINUTES)
.build(CacheLoader.asyncReloading(
new CacheLoader<PartitionValueCacheKey, HivePartitionValues>() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java
index 0982da418c..236aacf298 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java
@@ -53,7 +53,8 @@ public class PooledHiveMetaStoreClient {
private static final Logger LOG = LogManager.getLogger(PooledHiveMetaStoreClient.class);
private static final HiveMetaHookLoader DUMMY_HOOK_LOADER = t -> null;
- private static final short MAX_LIST_PARTITION_NUM = 10000;
+ // -1 means no limit on the partitions returned.
+ private static final short MAX_LIST_PARTITION_NUM = Config.max_hive_list_partition_num;
private Queue<CachedClient> clientPool = new LinkedList<>();
private final int poolSize;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/14: [Enhancement](storage engine) avoid deleting tablets on unused disk (#19010)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fffd4e4bc9082ad2b30ef2847ac06563d4b2038c
Author: Yueyang Zhan <40...@users.noreply.github.com>
AuthorDate: Thu May 4 15:15:43 2023 +0800
[Enhancement](storage engine) avoid deleting tablets on unused disk (#19010)
---
be/src/olap/storage_engine.cpp | 18 ++++--------------
be/src/olap/storage_engine.h | 2 +-
2 files changed, 5 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index a42f95e3e8..203ddafe0c 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -391,12 +391,8 @@ void StorageEngine::_start_disk_stat_monitor() {
}
_update_storage_medium_type_count();
- bool some_tablets_were_dropped = _delete_tablets_on_unused_root_path();
- // If some tablets were dropped, we should notify disk_state_worker_thread and
- // tablet_worker_thread (see TaskWorkerPool) to make them report to FE ASAP.
- if (some_tablets_were_dropped) {
- notify_listeners();
- }
+
+ _exit_if_too_many_disks_are_failed();
}
// TODO(lingbin): Should be in EnvPosix?
@@ -502,8 +498,7 @@ static bool too_many_disks_are_failed(uint32_t unused_num, uint32_t total_num) {
(unused_num * 100 / total_num > config::max_percentage_of_error_disk));
}
-bool StorageEngine::_delete_tablets_on_unused_root_path() {
- std::vector<TabletInfo> tablet_info_vec;
+void StorageEngine::_exit_if_too_many_disks_are_failed() {
uint32_t unused_root_path_num = 0;
uint32_t total_root_path_num = 0;
@@ -511,7 +506,7 @@ bool StorageEngine::_delete_tablets_on_unused_root_path() {
// TODO(yingchun): _store_map is only updated in main and ~StorageEngine, maybe we can remove it?
std::lock_guard<std::mutex> l(_store_lock);
if (_store_map.empty()) {
- return false;
+ return;
}
for (auto& it : _store_map) {
@@ -519,7 +514,6 @@ bool StorageEngine::_delete_tablets_on_unused_root_path() {
if (it.second->is_used()) {
continue;
}
- it.second->clear_tablets(&tablet_info_vec);
++unused_root_path_num;
}
}
@@ -531,10 +525,6 @@ bool StorageEngine::_delete_tablets_on_unused_root_path() {
<< ", total_disk_count=" << total_root_path_num;
exit(0);
}
-
- _tablet_manager->drop_tablets_on_error_root_path(tablet_info_vec);
- // If tablet_info_vec is not empty, means we have dropped some tablets.
- return !tablet_info_vec.empty();
}
void StorageEngine::stop() {
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 8701fd4fca..fa8c34f969 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -212,7 +212,7 @@ private:
Status _check_all_root_path_cluster_id();
Status _judge_and_update_effective_cluster_id(int32_t cluster_id);
- bool _delete_tablets_on_unused_root_path();
+ void _exit_if_too_many_disks_are_failed();
void _clean_unused_txns();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/14: [fix](JDBC) set jdbc parameters to compatible with both MySQL and Doris when reading boolean type (#19399)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9a51fc761efa464ec70dbf4853e74269d56c1ca1
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Tue May 9 13:53:17 2023 +0800
[fix](JDBC) set jdbc parameters to compatible with both MySQL and Doris when reading boolean type (#19399)
Fix errors when read boolean type from external doris cluster by jdbc catalog:
```
ERROR 1105 (HY000): errCode = 2, detailMessage = (172.16.10.11)[INTERNAL_ERROR]Fail to convert jdbc type of java.lang.Integer to doris type BOOL on column: deleted.
You need to check this column type between external table and doris table.
```
MySQL Types and Return Values for GetColumnTypeName and GetColumnClassName are presented in https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-type-conversions.html.
However when tinyInt1isBit=false, GetColumnClassName of MySQL returns java.lang.Boolean, while that of Doris returns java.lang.Integer. In order to be compatible with both MySQL and Doris, Jdbc params should set tinyInt1isBit=true&transformedBitIsBoolean=true
---
.../src/main/java/org/apache/doris/catalog/JdbcResource.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
index ad7d8dc0fd..ed09ff5861 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
@@ -279,10 +279,13 @@ public class JdbcResource extends Resource {
// `yearIsDateType` is a parameter of JDBC, and the default is true.
// We force the use of `yearIsDateType=false`
newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "yearIsDateType", "true", "false");
- // `tinyInt1isBit` is a parameter of JDBC, and the default is true.
- // We force the use of `tinyInt1isBit=false`, so that for mysql type tinyint,
- // it will convert to Doris tinyint, not bit.
- newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "tinyInt1isBit", "true", "false");
+ // MySQL Types and Return Values for GetColumnTypeName and GetColumnClassName
+ // are presented in https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-type-conversions.html
+ // However when tinyInt1isBit=false, GetColumnClassName of MySQL returns java.lang.Boolean,
+ // while that of Doris returns java.lang.Integer. In order to be compatible with both MySQL and Doris,
+ // Jdbc params should set tinyInt1isBit=true&transformedBitIsBoolean=true
+ newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "tinyInt1isBit", "true", "true");
+ newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "transformedBitIsBoolean", "true", "true");
// set useUnicode and characterEncoding to false and utf-8
newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "useUnicode", "false", "true");
newJdbcUrl = checkAndSetJdbcParam(newJdbcUrl, "characterEncoding", "utf-8");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 13/14: [minor](log) change debug log to info to observe the storage medium change #19529
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d9fe522cca8ddb19a5a65ee04be1217a44597853
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Fri May 12 11:02:55 2023 +0800
[minor](log) change debug log to info to observe the storage medium change #19529
When user set default_storage_medium to true, the storage medium of all partitions should be SSD,
and cooldown time should be 9999-12-31 23:59:59.
So that it won't change to HDD.
But looks like sometimes it still change to HDD.
So I change the debug log to info to observer it.
---
.../main/java/org/apache/doris/catalog/DataProperty.java | 13 +++++++++++--
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 6 ++++--
.../main/java/org/apache/doris/catalog/PartitionInfo.java | 4 ++--
3 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
index 8c24926dd8..b9bc6354c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
@@ -35,10 +35,12 @@ import java.io.IOException;
import java.util.Objects;
public class DataProperty implements Writable, GsonPostProcessable {
- public static final TStorageMedium DEFAULT_STORAGE_MEDIUM =
- "SSD".equalsIgnoreCase(Config.default_storage_medium) ? TStorageMedium.SSD : TStorageMedium.HDD;
+ public static final TStorageMedium DEFAULT_STORAGE_MEDIUM = "SSD".equalsIgnoreCase(Config.default_storage_medium)
+ ? TStorageMedium.SSD : TStorageMedium.HDD;
public static final long MAX_COOLDOWN_TIME_MS = 253402271999000L; // 9999-12-31 23:59:59
+ public static final DataProperty DEFAULT_HDD_DATA_PROPERTY = new DataProperty(TStorageMedium.HDD);
+
@SerializedName(value = "storageMedium")
private TStorageMedium storageMedium;
@SerializedName(value = "cooldownTimeMs")
@@ -56,6 +58,13 @@ public class DataProperty implements Writable, GsonPostProcessable {
this.storagePolicy = "";
}
+ public DataProperty(DataProperty other) {
+ this.storageMedium = other.storageMedium;
+ this.cooldownTimeMs = other.cooldownTimeMs;
+ this.storagePolicy = other.storagePolicy;
+ this.isMutable = other.isMutable;
+ }
+
/**
* DataProperty construction.
*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 970d86584d..306a52e39f 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3442,8 +3442,10 @@ public class Env {
DataProperty hddProperty = new DataProperty(TStorageMedium.HDD);
partitionInfo.setDataProperty(partition.getId(), hddProperty);
storageMediumMap.put(partitionId, TStorageMedium.HDD);
- LOG.debug("partition[{}-{}-{}] storage medium changed from SSD to HDD", dbId, tableId,
- partitionId);
+ LOG.info("partition[{}-{}-{}] storage medium changed from SSD to HDD. "
+ + "cooldown time: {}. current time: {}", dbId, tableId, partitionId,
+ TimeUtils.longToTimeString(dataProperty.getCooldownTimeMs()),
+ TimeUtils.longToTimeString(currentTimeMs));
// log
ModifyPartitionInfo info = new ModifyPartitionInfo(db.getId(), olapTable.getId(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index a18c5308c1..35ada94add 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -319,7 +319,7 @@ public class PartitionInfo implements Writable {
out.writeInt(idToDataProperty.size());
for (Map.Entry<Long, DataProperty> entry : idToDataProperty.entrySet()) {
out.writeLong(entry.getKey());
- if (entry.getValue().equals(new DataProperty(TStorageMedium.HDD))) {
+ if (entry.getValue().equals(DataProperty.DEFAULT_HDD_DATA_PROPERTY)) {
out.writeBoolean(true);
} else {
out.writeBoolean(false);
@@ -339,7 +339,7 @@ public class PartitionInfo implements Writable {
long partitionId = in.readLong();
boolean isDefaultHddDataProperty = in.readBoolean();
if (isDefaultHddDataProperty) {
- idToDataProperty.put(partitionId, new DataProperty(TStorageMedium.HDD));
+ idToDataProperty.put(partitionId, new DataProperty(DataProperty.DEFAULT_HDD_DATA_PROPERTY));
} else {
idToDataProperty.put(partitionId, DataProperty.read(in));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/14: [fix](metric) Fix bug for that register txn replica failed (#17855)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 970995b98a484a3e953617a10dc193301315e6ee
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Fri Mar 17 11:42:40 2023 +0800
[fix](metric) Fix bug for that register txn replica failed (#17855)
---
.../doris/load/routineload/KafkaTaskInfo.java | 9 -------
.../java/org/apache/doris/metric/MetricRepo.java | 13 ----------
.../org/apache/doris/planner/OlapTableSink.java | 11 --------
.../doris/transaction/DatabaseTransactionMgr.java | 30 +---------------------
.../doris/transaction/GlobalTransactionMgr.java | 9 ++-----
.../apache/doris/transaction/TransactionState.java | 8 ------
6 files changed, 3 insertions(+), 77 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java
index 9b9efbb5a4..fa4be7855a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java
@@ -20,7 +20,6 @@ package org.apache.doris.load.routineload;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Table;
-import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.thrift.TExecPlanFragmentParams;
@@ -32,10 +31,7 @@ import org.apache.doris.thrift.TRoutineLoadTask;
import org.apache.doris.thrift.TUniqueId;
import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
import com.google.gson.Gson;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
@@ -43,16 +39,11 @@ import java.util.Map;
import java.util.UUID;
public class KafkaTaskInfo extends RoutineLoadTaskInfo {
- private static final Logger LOG = LogManager.getLogger(KafkaTaskInfo.class);
-
private RoutineLoadManager routineLoadManager = Env.getCurrentEnv().getRoutineLoadManager();
// <partitionId, offset to be consumed>
private Map<Integer, Long> partitionIdToOffset;
- // Last fetched and cached latest partition offsets.
- private List<Pair<Integer, Long>> cachedPartitionWithLatestOffsets = Lists.newArrayList();
-
public KafkaTaskInfo(UUID id, long jobId, String clusterName,
long timeoutMs, Map<Integer, Long> partitionIdToOffset) {
super(id, jobId, clusterName, timeoutMs);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index 85c4479fbc..f38bc84e20 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -106,7 +106,6 @@ public final class MetricRepo {
public static Histogram HISTO_TXN_PUBLISH_LATENCY;
public static AutoMappedMetric<GaugeMetricImpl<Long>> DB_GAUGE_TXN_NUM;
public static AutoMappedMetric<GaugeMetricImpl<Long>> DB_GAUGE_PUBLISH_TXN_NUM;
- public static AutoMappedMetric<GaugeMetricImpl<Long>> DB_GAUGE_TXN_REPLICA_NUM;
public static LongCounterMetric COUNTER_ROUTINE_LOAD_ROWS;
public static LongCounterMetric COUNTER_ROUTINE_LOAD_RECEIVED_BYTES;
@@ -431,18 +430,6 @@ public final class MetricRepo {
DORIS_METRIC_REGISTER.addMetrics(publishTxnNum);
DB_GAUGE_PUBLISH_TXN_NUM = addLabeledMetrics("db",
() -> new GaugeMetricImpl<>("publish_txn_num", MetricUnit.NOUNIT, "number of publish transactions"));
-
- GaugeMetric<Long> txnReplicaNum = new GaugeMetric<Long>("txn_replica_num", MetricUnit.NOUNIT,
- "number of writing tablets in all running transactions") {
- @Override
- public Long getValue() {
- return Env.getCurrentGlobalTransactionMgr().getAllRunningTxnReplicaNum();
- }
- };
- DORIS_METRIC_REGISTER.addMetrics(txnReplicaNum);
- DB_GAUGE_TXN_REPLICA_NUM = addLabeledMetrics("db", () -> new GaugeMetricImpl<>("txn_replica_num",
- MetricUnit.NOUNIT, "number of writing tablets in all running transactions"));
-
COUNTER_ROUTINE_LOAD_ROWS = new LongCounterMetric("routine_load_rows", MetricUnit.ROWS,
"total rows of routine load");
DORIS_METRIC_REGISTER.addMetrics(COUNTER_ROUTINE_LOAD_ROWS);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
index 422d7e2ddd..d9f7fd7158 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -62,7 +62,6 @@ import org.apache.doris.thrift.TPaloNodesInfo;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletLocation;
import org.apache.doris.thrift.TUniqueId;
-import org.apache.doris.transaction.DatabaseTransactionMgr;
import com.google.common.base.Preconditions;
import com.google.common.collect.HashMultimap;
@@ -347,7 +346,6 @@ public class OlapTableSink extends DataSink {
TOlapTableLocationParam slaveLocationParam = new TOlapTableLocationParam();
// BE id -> path hash
Multimap<Long, Long> allBePathsMap = HashMultimap.create();
- int replicaNum = 0;
for (Long partitionId : partitionIds) {
Partition partition = table.getPartition(partitionId);
int quorum = table.getPartitionInfo().getReplicaAllocation(partition.getId()).getTotalReplicaNum() / 2 + 1;
@@ -377,7 +375,6 @@ public class OlapTableSink extends DataSink {
Lists.newArrayList(bePathsMap.keySet())));
}
allBePathsMap.putAll(bePathsMap);
- replicaNum += bePathsMap.size();
}
}
}
@@ -388,14 +385,6 @@ public class OlapTableSink extends DataSink {
if (!st.ok()) {
throw new DdlException(st.getErrorMsg());
}
- long dbId = tDataSink.getOlapTableSink().getDbId();
- long txnId = tDataSink.getOlapTableSink().getTxnId();
- try {
- DatabaseTransactionMgr mgr = Env.getCurrentGlobalTransactionMgr().getDatabaseTransactionMgr(dbId);
- mgr.registerTxnReplicas(txnId, replicaNum);
- } catch (Exception e) {
- LOG.error("register txn replica failed, txnId={}, dbId={}", txnId, dbId);
- }
return Arrays.asList(locationParam, slaveLocationParam);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 8a24e4361d..9c5c2d86a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -129,7 +129,6 @@ public class DatabaseTransactionMgr {
// count the number of running txns of database, except for the routine load txn
private volatile int runningTxnNums = 0;
- private volatile int runningTxnReplicaNums = 0;
// count only the number of running routine load txns of database
private volatile int runningRoutineLoadTxnNums = 0;
@@ -1115,20 +1114,6 @@ public class DatabaseTransactionMgr {
updateTxnLabels(transactionState);
}
- public void registerTxnReplicas(long txnId, int replicaNum) throws UserException {
- writeLock();
- try {
- TransactionState transactionState = idToRunningTransactionState.get(txnId);
- if (transactionState == null) {
- throw new UserException("running transaction not found, txnId=" + txnId);
- }
- transactionState.setReplicaNum(replicaNum);
- runningTxnReplicaNums += replicaNum;
- } finally {
- writeUnlock();
- }
- }
-
public int getRunningTxnNum() {
readLock();
try {
@@ -1138,21 +1123,8 @@ public class DatabaseTransactionMgr {
}
}
- public int getRunningTxnReplicaNum() {
- readLock();
- try {
- return runningTxnReplicaNums;
- } finally {
- readUnlock();
- }
- }
-
private void updateTxnLabels(TransactionState transactionState) {
- Set<Long> txnIds = labelToTxnIds.get(transactionState.getLabel());
- if (txnIds == null) {
- txnIds = Sets.newHashSet();
- labelToTxnIds.put(transactionState.getLabel(), txnIds);
- }
+ Set<Long> txnIds = labelToTxnIds.computeIfAbsent(transactionState.getLabel(), k -> Sets.newHashSet());
txnIds.add(transactionState.getTransactionId());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
index cb647ad1ef..f2bb98925b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
@@ -688,18 +688,13 @@ public class GlobalTransactionMgr implements Writable {
}
public long getAllRunningTxnNum() {
- return updateTxnMetric(databaseTransactionMgr -> Long.valueOf(databaseTransactionMgr.getRunningTxnNum()),
+ return updateTxnMetric(databaseTransactionMgr -> (long) databaseTransactionMgr.getRunningTxnNum(),
MetricRepo.DB_GAUGE_TXN_NUM);
}
- public long getAllRunningTxnReplicaNum() {
- return updateTxnMetric(databaseTransactionMgr -> Long.valueOf(databaseTransactionMgr.getRunningTxnReplicaNum()),
- MetricRepo.DB_GAUGE_TXN_REPLICA_NUM);
- }
-
public long getAllPublishTxnNum() {
return updateTxnMetric(
- databaseTransactionMgr -> Long.valueOf(databaseTransactionMgr.getCommittedTxnList().size()),
+ databaseTransactionMgr -> (long) databaseTransactionMgr.getCommittedTxnList().size(),
MetricRepo.DB_GAUGE_PUBLISH_TXN_NUM);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
index 7be1682f0a..b43d9cf1b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
@@ -498,14 +498,6 @@ public class TransactionState implements Writable {
return tableIdList;
}
- public int getReplicaNum() {
- return replicaNum;
- }
-
- public void setReplicaNum(int replicaNum) {
- this.replicaNum = replicaNum;
- }
-
public Map<Long, TableCommitInfo> getIdToTableCommitInfos() {
return idToTableCommitInfos;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/14: [enhancement](planner) fix unclear exception msg when create mv (#17537)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 122509487337659f5b9085a8207064c5a89ae58d
Author: mch_ucchi <41...@users.noreply.github.com>
AuthorDate: Sat Mar 25 11:36:40 2023 +0800
[enhancement](planner) fix unclear exception msg when create mv (#17537)
a materialized view's from clause can only be a single table and not sub-query, but the exception msg is npe. The pr change it to a clear msg.
---
.../java/org/apache/doris/analysis/CreateMaterializedViewStmt.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index a78528fdec..75e62afbad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -244,6 +244,10 @@ public class CreateMaterializedViewStmt extends DdlStmt {
throw new AnalysisException("The materialized view only support one table in from clause.");
}
TableName tableName = tableRefList.get(0).getName();
+ if (tableName == null) {
+ throw new AnalysisException("table in from clause is invalid, please check if it's single table "
+ + "and not sub-query");
+ }
baseIndexName = tableName.getTbl();
dbName = tableName.getDb();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/14: [fix](catalog) fix doris jdbc catalog largeint select error (#19407)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit de0139d5cc60e17829facefe9bc5612a74c743ab
Author: yongkang.zhong <zh...@qq.com>
AuthorDate: Tue May 9 17:34:48 2023 +0800
[fix](catalog) fix doris jdbc catalog largeint select error (#19407)
when I use mysql-jdbc 5.1.47 create a doris jdbc catalog, the largeint cannot select
When mysql-jdbc reads largeint, it will convert the format to string because it is too long
mysql> select `largeint` from type3;
ERROR 1105 (HY000): errCode = 2, detailMessage = (127.0.0.1)[INTERNAL_ERROR]Fail to convert jdbc type of java.lang.String to doris type LARGEINT on column: largeint. You need to check this column type between external table and doris table.
---
be/src/vec/exec/vjdbc_connector.cpp | 2 +-
.../src/main/java/org/apache/doris/udf/JdbcExecutor.java | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp
index 572be9e733..a01809e561 100644
--- a/be/src/vec/exec/vjdbc_connector.cpp
+++ b/be/src/vec/exec/vjdbc_connector.cpp
@@ -271,7 +271,7 @@ Status JdbcConnector::_check_type(SlotDescriptor* slot_desc, const std::string&
case TYPE_BIGINT:
case TYPE_LARGEINT: {
if (type_str != "java.lang.Long" && type_str != "java.math.BigDecimal" &&
- type_str != "java.math.BigInteger" &&
+ type_str != "java.math.BigInteger" && type_str != "java.lang.String" &&
type_str != "com.clickhouse.data.value.UnsignedInteger" &&
type_str != "com.clickhouse.data.value.UnsignedLong") {
return Status::InternalError(error_msg);
diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
index c8ba6ee1ad..8e20d8cc4c 100644
--- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
+++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
@@ -710,6 +710,20 @@ public class JdbcExecutor {
}
}
+ private void stringPutToBigInteger(Object[] column, boolean isNullable, int numRows, long nullMapAddr,
+ long columnAddr, int startRowForNullable) {
+ BigInteger[] data = new BigInteger[numRows];
+ for (int i = 0; i < numRows; i++) {
+ if (column[i] == null) {
+ data[i] = null;
+ UdfUtils.UNSAFE.putByte(nullMapAddr + i, (byte) 1);
+ } else {
+ data[i] = new BigInteger((String) column[i]);
+ }
+ }
+ copyBatchDecimalResult(data, isNullable, numRows, columnAddr, 16, startRowForNullable);
+ }
+
private void clickHouseUInt64ToLong(Object[] column, boolean isNullable, int numRows, long nullMapAddr,
long columnAddr, int startRowForNullable) {
if (isNullable) {
@@ -741,6 +755,8 @@ public class JdbcExecutor {
bigDecimalPutToBigInteger(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex);
} else if (column[firstNotNullIndex] instanceof BigInteger) {
bigIntegerPutToByte(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex);
+ } else if (column[firstNotNullIndex] instanceof String) {
+ stringPutToBigInteger(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex);
} else if (column[firstNotNullIndex] instanceof com.clickhouse.data.value.UnsignedLong) {
clickHouseUInt64ToLong(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 14/14: [branch1.2](fix) fix compilation error after cherry-pick
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 77b84638861e3bfe2fa35b7b1f17fb8274cfad70
Author: morningman <mo...@163.com>
AuthorDate: Sat May 13 16:06:58 2023 +0800
[branch1.2](fix) fix compilation error after cherry-pick
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 6 ++----
fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java | 1 -
2 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 9e3c3aac3e..662c723032 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1900,12 +1900,10 @@ public class Config extends ConfigBase {
@ConfField(mutable = false, masterOnly = false)
public static long max_hive_partition_cache_num = 100000;
- @ConfField(mutable = false, masterOnly = false, description = {"Hive表到分区名列表缓存的最大数量。",
- "Max cache number of hive table to partition names list."})
+ @ConfField(mutable = false, masterOnly = false)
public static long max_hive_table_catch_num = 1000;
- @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量,-1代表没有限制。",
- "Max number of hive partition values to return while list partitions, -1 means no limitation."})
+ @ConfField(mutable = false, masterOnly = false)
public static short max_hive_list_partition_num = -1;
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
index b9bc6354c2..03af60f6c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
@@ -62,7 +62,6 @@ public class DataProperty implements Writable, GsonPostProcessable {
this.storageMedium = other.storageMedium;
this.cooldownTimeMs = other.cooldownTimeMs;
this.storagePolicy = other.storagePolicy;
- this.isMutable = other.isMutable;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/14: [fix](fe)havingClause should be substituted the same way as resultExprs (#19261)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9be8263050db9c52337052f82fd3adab8d3488db
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Fri May 5 18:03:43 2023 +0800
[fix](fe)havingClause should be substituted the same way as resultExprs (#19261)
substituted havingClause in the same way as resultExprs to prevent " HAVING clause not produced by aggregation output" error
---
.../src/main/java/org/apache/doris/analysis/SelectStmt.java | 8 ++++++--
regression-test/data/correctness_p0/test_group_having_alias.out | 3 +++
.../suites/correctness_p0/test_group_having_alias.groovy | 3 +++
3 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 5884ea7d0f..9e36a5a51a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -1155,9 +1155,13 @@ public class SelectStmt extends QueryStmt {
countAllMap = ExprSubstitutionMap.compose(multiDistinctAggMap, countAllMap, analyzer);
List<Expr> substitutedAggs =
Expr.substituteList(aggExprs, countAllMap, analyzer, false);
- // the resultExprs must substitute in the same way as aggExprs
- // then resultExprs can be substitute correctly using combinedSmap
+ // the resultExprs and havingClause must substitute in the same way as aggExprs
+ // then resultExprs and havingClause can be substitute correctly using combinedSmap
resultExprs = Expr.substituteList(resultExprs, countAllMap, analyzer, false);
+ if (havingClauseAfterAnaylzed != null) {
+ havingClauseAfterAnaylzed =
+ havingClauseAfterAnaylzed.substitute(countAllMap, analyzer, false);
+ }
aggExprs.clear();
TreeNode.collect(substitutedAggs, Expr.isAggregatePredicate(), aggExprs);
diff --git a/regression-test/data/correctness_p0/test_group_having_alias.out b/regression-test/data/correctness_p0/test_group_having_alias.out
index 1c4feeeecb..4ba36af336 100644
--- a/regression-test/data/correctness_p0/test_group_having_alias.out
+++ b/regression-test/data/correctness_p0/test_group_having_alias.out
@@ -36,3 +36,6 @@
-- !case5 --
2 3
+-- !case_cir2273 --
+2.5
+
diff --git a/regression-test/suites/correctness_p0/test_group_having_alias.groovy b/regression-test/suites/correctness_p0/test_group_having_alias.groovy
index 8429be8920..c74fe96420 100644
--- a/regression-test/suites/correctness_p0/test_group_having_alias.groovy
+++ b/regression-test/suites/correctness_p0/test_group_having_alias.groovy
@@ -133,5 +133,8 @@
qt_case3 """ SELECT id, v1-2 as v, sum(v2) v2 FROM test_having_alias_tb GROUP BY id,v having(v>0 AND sum(v2)>1) ORDER BY id,v; """
qt_case4 """ SELECT id, v1-2 as v, sum(v2) vsum FROM test_having_alias_tb GROUP BY id,v having(v>0 AND vsum>1) ORDER BY id,v; """
qt_case5 """ SELECT id, max(v1) v1 FROM test_having_alias_tb GROUP BY 1 having count(distinct v1)>1 ORDER BY id; """
+ qt_case_cir2273 """ select sum( id * 0.5 ) val from test_having_alias_tb having val > 0; """
+
sql """ DROP TABLE IF EXISTS `test_having_alias_tb`; """
}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 11/14: [Bug](scan) forbiden push down in predicate when in_state->use_set is false (#19471)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit abc4483d0089c83925c03f73fc541cada6a3aff3
Author: Pxl <px...@qq.com>
AuthorDate: Wed May 10 11:12:20 2023 +0800
[Bug](scan) forbiden push down in predicate when in_state->use_set is false (#19471)
forbiden push down in predicate when in_state->use_set is false
---
be/src/vec/exec/scan/vscan_node.cpp | 12 ++++++++++++
regression-test/data/query_p0/sql_functions/test_in_expr.out | 10 ++++++++++
.../suites/query_p0/sql_functions/test_in_expr.groovy | 11 +++++++++++
3 files changed, 33 insertions(+)
diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp
index 63d40d4952..ce80396fa6 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -662,6 +662,12 @@ Status VScanNode::_normalize_in_and_eq_predicate(VExpr* expr, VExprContext* expr
InState* state = reinterpret_cast<InState*>(
expr_ctx->fn_context(pred->fn_context_index())
->get_function_state(FunctionContext::FRAGMENT_LOCAL));
+
+ // xx in (col, xx, xx) should not be push down
+ if (!state->use_set) {
+ return Status::OK();
+ }
+
iter = state->hybrid_set->begin();
}
@@ -737,6 +743,12 @@ Status VScanNode::_normalize_not_in_and_not_eq_predicate(VExpr* expr, VExprConte
InState* state = reinterpret_cast<InState*>(
expr_ctx->fn_context(pred->fn_context_index())
->get_function_state(FunctionContext::FRAGMENT_LOCAL));
+
+ // xx in (col, xx, xx) should not be push down
+ if (!state->use_set) {
+ return Status::OK();
+ }
+
HybridSetBase::IteratorBase* iter = state->hybrid_set->begin();
auto fn_name = std::string("");
if (!is_fixed_range && state->null_in_set) {
diff --git a/regression-test/data/query_p0/sql_functions/test_in_expr.out b/regression-test/data/query_p0/sql_functions/test_in_expr.out
index 31d6bb5b1a..c7f58848a5 100644
--- a/regression-test/data/query_p0/sql_functions/test_in_expr.out
+++ b/regression-test/data/query_p0/sql_functions/test_in_expr.out
@@ -53,3 +53,13 @@ a
b
d
+-- !select --
+2
+3
+4
+
+-- !select --
+2
+3
+4
+
diff --git a/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy b/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy
index 01cacef28e..a2424e4d82 100644
--- a/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy
@@ -115,4 +115,15 @@ suite("test_in_expr", "query") {
sql """DROP TABLE IF EXISTS ${nullTableName}"""
sql """DROP TABLE IF EXISTS ${notNullTableName}"""
+ // from https://github.com/apache/doris/issues/19374
+ sql """DROP TABLE IF EXISTS t11"""
+ sql """
+ CREATE TABLE t11(c0 CHAR(109) NOT NULL) DISTRIBUTED BY HASH (c0) BUCKETS 13 PROPERTIES ("replication_num" = "1");
+ """
+ sql """
+ insert into t11 values ('1'), ('1'), ('2'), ('2'), ('3'), ('4');
+ """
+
+ qt_select "select t11.c0 from t11 group by t11.c0 having not ('1' in (t11.c0)) order by t11.c0;"
+ qt_select "select t11.c0 from t11 group by t11.c0 having ('1' not in (t11.c0)) order by t11.c0;"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/14: [fix](planner) insubquery should always be converted to semi or anti join (#19240)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 49e8550e9aec93d3c8fdccdebc57ce14308bd534
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Thu May 4 11:16:18 2023 +0800
[fix](planner) insubquery should always be converted to semi or anti join (#19240)
---
.../src/main/java/org/apache/doris/analysis/StmtRewriter.java | 10 ++++++----
.../suites/correctness_p0/test_join_without_condition.groovy | 6 ++++++
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
index eae4c6e215..c0b3a14ae6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
@@ -808,11 +808,13 @@ public class StmtRewriter {
if (expr instanceof ExistsPredicate) {
joinOp = ((ExistsPredicate) expr).isNotExists() ? JoinOperator.LEFT_ANTI_JOIN :
JoinOperator.LEFT_SEMI_JOIN;
- } else if (expr instanceof InPredicate && joinConjunct instanceof FunctionCallExpr
- && (((FunctionCallExpr) joinConjunct).getFnName().getFunction()
- .equalsIgnoreCase(BITMAP_CONTAINS))) {
+ } else if (expr instanceof InPredicate && !(joinConjunct instanceof BitmapFilterPredicate)) {
joinOp = ((InPredicate) expr).isNotIn() ? JoinOperator.LEFT_ANTI_JOIN : JoinOperator.LEFT_SEMI_JOIN;
- isInBitmap = true;
+ if ((joinConjunct instanceof FunctionCallExpr
+ && (((FunctionCallExpr) joinConjunct).getFnName().getFunction()
+ .equalsIgnoreCase(BITMAP_CONTAINS)))) {
+ isInBitmap = true;
+ }
} else {
joinOp = JoinOperator.CROSS_JOIN;
// We can equal the aggregate subquery using a cross join. All conjuncts
diff --git a/regression-test/suites/correctness_p0/test_join_without_condition.groovy b/regression-test/suites/correctness_p0/test_join_without_condition.groovy
index c56144757a..88f86e23c7 100644
--- a/regression-test/suites/correctness_p0/test_join_without_condition.groovy
+++ b/regression-test/suites/correctness_p0/test_join_without_condition.groovy
@@ -63,6 +63,12 @@ suite("test_join_without_condition") {
order by a.a, b.a;
"""
+ explain {
+ sql("select * from (select 1 id) t where (1 in (select a from test_join_without_condition_a))")
+ notContains "CROSS JOIN"
+ contains "LEFT SEMI JOIN"
+ }
+
sql """
drop table if exists test_join_without_condition_a;
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org