You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/22 15:41:24 UTC
[doris] branch branch-1.2-lts updated (3255173e66 -> 401a233caf)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 3255173e66 [refactor](fe) remove type related class to fe-common to reduce java-udf jar size (#15808)
new 2ab07e5153 [fix](planner) extract common factor rule should consider not only where predicate (#16110)
new 401a233caf [deps](libhdfs3) update to 2.3.6 to fix kms aes 256 bug (#16127)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dist/LICENSE-dist.txt | 57 +++++++++++-----------
.../doris/rewrite/ExtractCommonFactorsRule.java | 3 +-
.../ExtractCommonFactorsRuleFunctionTest.java | 24 +++++++++
thirdparty/CHANGELOG.md | 15 ++++++
thirdparty/vars.sh | 8 +--
5 files changed, 72 insertions(+), 35 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: [deps](libhdfs3) update to 2.3.6 to fix kms aes 256 bug (#16127)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 401a233cafdf3378f674431ae9268dc29d883c92
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Sun Jan 22 07:18:35 2023 +0800
[deps](libhdfs3) update to 2.3.6 to fix kms aes 256 bug (#16127)
update libhdfs3 to 2.3.6 to fix kms aes 256 bug.
And update the licences and changelog
---
dist/LICENSE-dist.txt | 57 ++++++++++++++++++++++++-------------------------
thirdparty/CHANGELOG.md | 15 +++++++++++++
thirdparty/vars.sh | 8 +++----
3 files changed, 47 insertions(+), 33 deletions(-)
diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt
index 837cfc0462..093d2b33bf 100644
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@@ -680,20 +680,19 @@ The Apache Software License, Version 2.0
* Apache Directory LDAP API Utilities:
- org.apache.directory.api:api-util:1.0.0-M20 (http://directory.apache.org/api-parent/api-util/)
* Apache Hadoop Amazon Web Services support:
- - org.apache.hadoop:hadoop-aws:2.7.3 (no url defined)
- - org.apache.hadoop:hadoop-aws:2.8.0 (no url defined)
+ - org.apache.hadoop:hadoop-aws:3.3.3 (no url defined)
* Apache Hadoop Annotations:
- - org.apache.hadoop:hadoop-annotations:2.10.2 (no url defined)
+ - org.apache.hadoop:hadoop-annotations:3.3.3 (no url defined)
* Apache Hadoop Auth:
- - org.apache.hadoop:hadoop-auth:2.10.2 (no url defined)
+ - org.apache.hadoop:hadoop-auth:3.3.3 (no url defined)
* Apache Hadoop Client:
- - org.apache.hadoop:hadoop-client:2.10.2 (no url defined)
+ - org.apache.hadoop:hadoop-client:3.3.3 (no url defined)
* Apache Hadoop Common:
- - org.apache.hadoop:hadoop-common:2.10.2 (no url defined)
+ - org.apache.hadoop:hadoop-common:3.3.3 (no url defined)
* Apache Hadoop HDFS:
- - org.apache.hadoop:hadoop-hdfs:2.10.2 (no url defined)
+ - org.apache.hadoop:hadoop-hdfs:3.3.3 (no url defined)
* Apache Hadoop HDFS Client:
- - org.apache.hadoop:hadoop-hdfs-client:2.10.2 (no url defined)
+ - org.apache.hadoop:hadoop-hdfs-client:3.3.3 (no url defined)
* Apache HttpClient:
- org.apache.httpcomponents:httpclient:4.5.13 (http://hc.apache.org/httpcomponents-client)
* Apache HttpCore:
@@ -881,33 +880,33 @@ The Apache Software License, Version 2.0
* velocity:
- org.apache.velocity:velocity-engine-core:2.3 (https://velocity.apache.org/engine/2.3/)
* Hive Common:
- - org.apache.hive:hive-common:2.3.7 (https://hive.apache.org/hive-common)
+ - org.apache.hive:hive-common:3.1.3 (https://hive.apache.org/hive-common)
* Hive Llap Client:
- - org.apache.hive:hive-llap-client:2.3.7 (https://hive.apache.org/hive-llap-client)
+ - org.apache.hive:hive-llap-client:3.1.3 (https://hive.apache.org/hive-llap-client)
* Hive Llap Common:
- - org.apache.hive:hive-llap-common:2.3.7 (https://hive.apache.org/hive-llap-common)
+ - org.apache.hive:hive-llap-common:3.1.3 (https://hive.apache.org/hive-llap-common)
* Hive Llap Tez:
- - org.apache.hive:hive-llap-tez:2.3.7 (https://hive.apache.org/hive-llap-tez)
+ - org.apache.hive:hive-llap-tez:3.1.3 (https://hive.apache.org/hive-llap-tez)
* Hive Metastore:
- - org.apache.hive:hive-metastore:2.3.7 (https://hive.apache.org/hive-metastore)
+ - org.apache.hive:hive-metastore:3.1.3 (https://hive.apache.org/hive-metastore)
* Hive Query Language:
- - org.apache.hive:hive-exec:2.3.7 (https://hive.apache.org/hive-exec)
+ - org.apache.hive:hive-exec:3.1.3 (https://hive.apache.org/hive-exec)
* Hive Serde:
- - org.apache.hive:hive-serde:2.3.7 (https://hive.apache.org/hive-serde)
+ - org.apache.hive:hive-serde:3.1.3 (https://hive.apache.org/hive-serde)
* Hive Service RPC:
- - org.apache.hive:hive-service-rpc:2.3.7 (https://hive.apache.org/hive-service-rpc)
+ - org.apache.hive:hive-service-rpc:3.1.3 (https://hive.apache.org/hive-service-rpc)
* Hive Shims:
- - org.apache.hive:hive-shims:2.3.7 (https://hive.apache.org/hive-shims)
+ - org.apache.hive:hive-shims:3.1.3 (https://hive.apache.org/hive-shims)
* Hive Shims 0.23:
- - org.apache.hive.shims:hive-shims-0.23:2.3.7 (https://hive.apache.org/hive-shims-0.23)
+ - org.apache.hive.shims:hive-shims-0.23:3.1.3 (https://hive.apache.org/hive-shims-0.23)
* Hive Shims Common:
- - org.apache.hive.shims:hive-shims-common:2.3.7 (https://hive.apache.org/hive-shims-common)
+ - org.apache.hive.shims:hive-shims-common:3.1.3 (https://hive.apache.org/hive-shims-common)
* Hive Shims Scheduler:
- - org.apache.hive.shims:hive-shims-scheduler:2.3.7 (https://hive.apache.org/hive-shims-scheduler)
+ - org.apache.hive.shims:hive-shims-scheduler:3.1.3 (https://hive.apache.org/hive-shims-scheduler)
* Hive Storage API:
- org.apache.hive:hive-storage-api:2.4.0 (https://www.apache.org/hive-storage-api/)
* Hive Vector-Code-Gen Utilities:
- - org.apache.hive:hive-vector-code-gen:2.3.7 (https://hive.apache.org/hive-vector-code-gen)
+ - org.apache.hive:hive-vector-code-gen:3.1.3 (https://hive.apache.org/hive-vector-code-gen)
* HttpClient:
- commons-httpclient:commons-httpclient:3.0.1 (http://jakarta.apache.org/commons/httpclient/)
- commons-httpclient:commons-httpclient:3.1 (http://jakarta.apache.org/httpcomponents/httpclient-3.x/)
@@ -947,15 +946,15 @@ The Apache Software License, Version 2.0
* Jackson datatype: jdk8:
- com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.1 (https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8)
* Jackson module: JAXB Annotations:
- - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.1 (https://github.com/FasterXML/jackson-modules-base)
+ - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.3 (https://github.com/FasterXML/jackson-modules-base)
* Jackson module: Paranamer:
- com.fasterxml.jackson.module:jackson-module-paranamer:2.7.9 (https://github.com/FasterXML/jackson-modules-base)
* Jackson-annotations:
- - com.fasterxml.jackson.core:jackson-annotations:2.12.1 (http://github.com/FasterXML/jackson)
+ - com.fasterxml.jackson.core:jackson-annotations:2.12.3 (http://github.com/FasterXML/jackson)
* Jackson-core:
- - com.fasterxml.jackson.core:jackson-core:2.12.1 (https://github.com/FasterXML/jackson-core)
+ - com.fasterxml.jackson.core:jackson-core:2.12.3 (https://github.com/FasterXML/jackson-core)
* Jackson-dataformat-YAML:
- - com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.1 (https://github.com/FasterXML/jackson-dataformats-text)
+ - com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3 (https://github.com/FasterXML/jackson-dataformats-text)
* Jackson-module-parameter-names:
- com.fasterxml.jackson.module:jackson-module-parameter-names:2.13.1 (https://github.com/FasterXML/jackson-modules-java8/jackson-module-parameter-names)
* Jakarta Bean Validation API:
@@ -1530,10 +1529,10 @@ The Apache Software License, Version 2.0
* aws sdk: 1.9.211
* benchmark: 1.5.6
* simdjson: 3.0.1
- * libhdfs3: 2.3.0
- * libhdfs3: commit 5fccd36
+ * libhdfs3: 2.3.6
* opentelemetry-proto: 0.18.0
* opentelemetry-cpp: 1.4.0
+ * clucenc: 2.4.4
The MIT License -- licenses/LICENSE-MIT.txt
* datatables: 1.12.1
@@ -1548,7 +1547,7 @@ LGPL -- licenes/LICENSE-LGPL.txt
Other dependencies:
* libevent: 2.1.12 -- license/LICENSE-libevent.txt
- * openssl: 1.1.1m -- license/LICENSE-openssl.txt
+ * openssl: 1.1.1s -- license/LICENSE-openssl.txt
* gflag: 2.2.2 -- license/LICENSE-gflag.txt
* glog: 0.4.0 -- license/LICENSE-glog.txt
* gtest: 1.11.0 -- license/LICENSE-gtest.txt
@@ -1569,7 +1568,7 @@ Other dependencies:
* librdkafka: 1.8.2 -- license/LICENSE-librdkafka.txt
* zstd: 1.5.2 -- license/LICENSE-zstd.txt
* brotli: 1.0.9 -- license/LICENSE-brotli.txt
- * bitshuffle: 0.3.5 -- license/LICENSE-bigshuffle.txt
+ * bitshuffle: 0.5.1 -- license/LICENSE-bigshuffle.txt
* fmt: 7.1.3 -- license/LICENSE-fmt.txt
* jemalloc: 5.2.1 -- license/LICENSE-jemolloc.txt
* lzma@master -- license/LICENSE-lzma.txt
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index b8a70fb55f..30fd8587c9 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,21 @@
This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
+## v20230120
+- Modified: libhdfs3 2.3.5 -> 2.3.6
+
+## v20230117
+- Modified: bitshuffle 0.3.5 -> 0.5.1
+
+## v20230112
+- Added: clucene 2.4.4
+
+## v20230111
+- Fixed: libgsasl enable GSSAPI
+
+## v20230110
+- Modified: libhdfs3 2.3.4 -> 2.3.5
+
## v20230105
- Modified: openssl 1.1.1m -> 1.1.1s fix CVE-2022-1292
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 3e169b5c74..7a9e2fa4ee 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -357,10 +357,10 @@ KRB5_SOURCE="krb5-1.19"
KRB5_MD5SUM="aaf18447a5a014aa3b7e81814923f4c9"
# hdfs3
-HDFS3_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/libhdfs3-v2.3.5.tar.gz"
-HDFS3_NAME="doris-thirdparty-libhdfs3-v2.3.5.tar.gz"
-HDFS3_SOURCE="doris-thirdparty-libhdfs3-v2.3.5"
-HDFS3_MD5SUM="8da623120add76f5595e5978b123e157"
+HDFS3_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/libhdfs3-v2.3.6.tar.gz"
+HDFS3_NAME="doris-thirdparty-libhdfs3-v2.3.6.tar.gz"
+HDFS3_SOURCE="doris-thirdparty-libhdfs3-v2.3.6"
+HDFS3_MD5SUM="179e492ee71b6dc484e8f44bc2865de9"
#libdivide
LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/5.0.tar.gz"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [fix](planner) extract common factor rule should consider not only where predicate (#16110)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2ab07e515343c2da0ac6a0818ea72acda8431b4a
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Fri Jan 20 14:53:48 2023 +0800
[fix](planner) extract common factor rule should consider not only where predicate (#16110)
This PR #14381 limit the `ExtractCommonFactorsRule` to handle only `WHERE` predicate,
but the predicate in `ON` clause should also be considered. Such as:
```
CREATE TABLE `nation` (
`n_nationkey` int(11) NOT NULL,
`n_name` varchar(25) NOT NULL,
`n_regionkey` int(11) NOT NULL,
`n_comment` varchar(152) NULL
)
DUPLICATE KEY(`n_nationkey`)
DISTRIBUTED BY HASH(`n_nationkey`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
select * from
nation n1 join nation n2
on (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY')
or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE')
```
There should be predicates:
```
PREDICATES: `n1`.`n_name` IN ('FRANCE', 'GERMANY')
PREDICATES: `n2`.`n_name` IN ('FRANCE', 'GERMANY')
```
On each scan node.
This PR fix this issue by removing the limit of `ExtractCommonFactorsRule`
---
.../doris/rewrite/ExtractCommonFactorsRule.java | 3 +--
.../ExtractCommonFactorsRuleFunctionTest.java | 24 ++++++++++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExtractCommonFactorsRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExtractCommonFactorsRule.java
index 3c808c6100..74130d52af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExtractCommonFactorsRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExtractCommonFactorsRule.java
@@ -29,7 +29,6 @@ import org.apache.doris.analysis.TableName;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.planner.PlanNode;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.rewrite.ExprRewriter.ClauseType;
import com.google.common.base.Preconditions;
import com.google.common.collect.BoundType;
@@ -169,7 +168,7 @@ public class ExtractCommonFactorsRule implements ExprRewriteRule {
}
// 3. find merge cross the clause
- if (analyzer.getContext() != null && clauseType == ClauseType.WHERE_CLAUSE
+ if (analyzer.getContext() != null
&& analyzer.getContext().getSessionVariable().isExtractWideRangeExpr()) {
Expr wideCommonExpr = findWideRangeExpr(clearExprs);
if (wideCommonExpr != null && !(wideCommonExpr instanceof CompoundPredicate
diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java
index 588df5c6a6..2f994be85b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java
@@ -41,6 +41,7 @@ public class ExtractCommonFactorsRuleFunctionTest {
private static final String TABLE_NAME_1 = "tb1";
private static final String TABLE_NAME_2 = "tb2";
private static final String TABLE_NAME_3 = "tb3";
+ private static final String TABLE_NAME_4 = "nation";
@BeforeClass
public static void beforeClass() throws Exception {
@@ -61,6 +62,18 @@ public class ExtractCommonFactorsRuleFunctionTest {
+ " (k1 tinyint, k2 smallint, k3 int, k4 bigint, k5 largeint, k6 date, k7 datetime, k8 float, k9 double) "
+ "distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
dorisAssert.withTable(createTableSQL);
+ createTableSQL = "CREATE TABLE " + DB_NAME + "." + TABLE_NAME_4 + "(\n"
+ + " `n_nationkey` int(11) NOT NULL,\n"
+ + " `n_name` varchar(25) NOT NULL,\n"
+ + " `n_regionkey` int(11) NOT NULL,\n"
+ + " `n_comment` varchar(152) NULL\n"
+ + ")\n"
+ + "DUPLICATE KEY(`n_nationkey`)\n"
+ + "DISTRIBUTED BY HASH(`n_nationkey`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_allocation\" = \"tag.location.default: 1\"\n"
+ + ");";
+ dorisAssert.withTable(createTableSQL);
}
@AfterClass
@@ -311,4 +324,15 @@ public class ExtractCommonFactorsRuleFunctionTest {
String sql = "select * from tb3 where k9 like '%4%';";
LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
}
+
+ @Test
+ public void testExtractCommonFactorsWithOnClause() throws Exception {
+ String sql = "select * from\n"
+ + "db1.nation n1 join db1.nation n2\n"
+ + "on (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY')\n"
+ + "or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE')";
+ String explainStr = dorisAssert.query(sql).explainQuery();
+ Assert.assertTrue(explainStr.contains("PREDICATES: `n1`.`n_name` IN ('FRANCE', 'GERMANY')"));
+ Assert.assertTrue(explainStr.contains("PREDICATES: `n2`.`n_name` IN ('FRANCE', 'GERMANY')"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org