You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/22 07:55:44 UTC
[doris] branch branch-1.2-lts updated (9c0239ef40 -> 258db947af)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 9c0239ef40 [fix](jdbc) fix create table like table of jdbc error (#15179)
new 84025ad894 [fix](multi-catalog) can't show databases when creating a new user in external catalog (#15204)
new 258db947af [bug](jdbc) fix error of jdbc with datetime type in oracle (#15205)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/en/docs/admin-manual/config/fe-config.md | 9 +++++++
.../docs/ecosystem/external-table/jdbc-of-doris.md | 2 +-
docs/zh-CN/docs/admin-manual/config/fe-config.md | 17 +++++++++++++
.../docs/ecosystem/external-table/jdbc-of-doris.md | 2 +-
.../java/org/apache/doris/analysis/ShowDbStmt.java | 3 ++-
.../main/java/org/apache/doris/common/Config.java | 7 ++++++
.../org/apache/doris/planner/JdbcScanNode.java | 2 +-
.../org/apache/doris/planner/OdbcScanNode.java | 29 ++++++++++++++++------
.../java/org/apache/doris/qe/ShowExecutor.java | 3 ++-
.../java/org/apache/doris/qe/ShowExecutorTest.java | 16 ++++++++++++
.../external_catalog_p0/hive/test_hive_other.out | 5 ++++
.../hive/test_hive_other.groovy | 11 ++++++++
12 files changed, 93 insertions(+), 13 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [fix](multi-catalog) can't show databases when creating a new user in external catalog (#15204)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 84025ad8942d5ea60df4a040124d6becc1d71700
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Wed Dec 21 08:58:06 2022 +0800
[fix](multi-catalog) can't show databases when creating a new user in external catalog (#15204)
Fix bug: A new user with grants to access external catalog can't show databases.
---
.../main/java/org/apache/doris/analysis/ShowDbStmt.java | 3 ++-
.../src/main/java/org/apache/doris/qe/ShowExecutor.java | 3 ++-
.../test/java/org/apache/doris/qe/ShowExecutorTest.java | 16 ++++++++++++++++
.../data/external_catalog_p0/hive/test_hive_other.out | 5 +++++
.../external_catalog_p0/hive/test_hive_other.groovy | 11 +++++++++++
5 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java
index e29605a640..f38199042e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java
@@ -68,6 +68,7 @@ public class ShowDbStmt extends ShowStmt {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
+ this.catalogName = this.catalogName == null ? analyzer.getDefaultCatalog() : this.catalogName;
}
@Override
@@ -102,7 +103,7 @@ public class ShowDbStmt extends ShowStmt {
if (pattern != null) {
sb.append(" LIKE '").append(pattern).append("'");
}
- if (catalogName != null) {
+ if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalogName)) {
sb.append(" FROM ").append(catalogName);
}
return sb.toString();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 34360271ba..3969603186 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -707,7 +707,8 @@ public class ShowExecutor {
continue;
}
- if (!Env.getCurrentEnv().getAuth().checkDbPriv(ConnectContext.get(), fullName, PrivPredicate.SHOW)) {
+ if (!Env.getCurrentEnv().getAuth().checkDbPriv(ConnectContext.get(), showDbStmt.getCatalogName(),
+ fullName, PrivPredicate.SHOW)) {
continue;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
index 2368b1b3d2..1b4a3afd5a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
@@ -34,6 +34,7 @@ import org.apache.doris.analysis.ShowTableStmt;
import org.apache.doris.analysis.ShowVariablesStmt;
import org.apache.doris.analysis.ShowViewStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
@@ -260,6 +261,7 @@ public class ShowExecutorTest {
ctx.setEnv(AccessTestUtil.fetchAdminCatalog());
ctx.setQualifiedUser("testCluster:testUser");
ctx.setCluster("testCluster");
+ ctx.setCurrentUserIdentity(UserIdentity.ROOT);
new Expectations(ctx) {
{
@@ -272,7 +274,14 @@ public class ShowExecutorTest {
@Test
public void testShowDb() throws AnalysisException {
+ Analyzer analyzer = AccessTestUtil.fetchAdminAnalyzer(false);
ShowDbStmt stmt = new ShowDbStmt(null);
+ try {
+ stmt.analyze(analyzer);
+ } catch (UserException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
ShowExecutor executor = new ShowExecutor(ctx, stmt);
ShowResultSet resultSet = executor.execute();
@@ -301,7 +310,14 @@ public class ShowExecutorTest {
@Test
public void testShowDbPriv() throws AnalysisException {
+ Analyzer analyzer = AccessTestUtil.fetchAdminAnalyzer(false);
ShowDbStmt stmt = new ShowDbStmt(null);
+ try {
+ stmt.analyze(analyzer);
+ } catch (UserException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
ShowExecutor executor = new ShowExecutor(ctx, stmt);
ctx.setEnv(AccessTestUtil.fetchBlockCatalog());
executor.execute();
diff --git a/regression-test/data/external_catalog_p0/hive/test_hive_other.out b/regression-test/data/external_catalog_p0/hive/test_hive_other.out
index 8d44514ed7..05c25b0e19 100644
--- a/regression-test/data/external_catalog_p0/hive/test_hive_other.out
+++ b/regression-test/data/external_catalog_p0/hive/test_hive_other.out
@@ -1,4 +1,9 @@
-- This file is automatically generated. You should know what you did if you want to edit this
+-- !ext_catalog_grants --
+default
+tpch1_orc
+tpch1_parquet
+
-- !q24 --
zhangsan 1
lisi 1
diff --git a/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy b/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy
index 7f305b3714..c6b67cd70d 100644
--- a/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy
+++ b/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy
@@ -63,6 +63,17 @@ suite("test_hive_other", "p0") {
'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}'
);
"""
+
+ // test user's grants on external catalog
+ sql """drop user if exists ext_catalog_user"""
+ sql """create user ext_catalog_user identified by '12345'"""
+ sql """grant all on internal.${context.config.defaultDb}.* to ext_catalog_user"""
+ sql """grant all on ${catalog_name}.*.* to ext_catalog_user"""
+ connect(user = 'ext_catalog_user', password = '12345', url = context.config.jdbcUrl) {
+ order_qt_ext_catalog_grants """show databases from ${catalog_name}"""
+ }
+ sql """drop user ext_catalog_user"""
+
sql """switch ${catalog_name}"""
sql """use `default`"""
// order_qt_show_tables """show tables"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: [bug](jdbc) fix error of jdbc with datetime type in oracle (#15205)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 258db947af5c0d010633b04d8d1280b3e33d5907
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Tue Dec 20 22:05:55 2022 +0800
[bug](jdbc) fix error of jdbc with datetime type in oracle (#15205)
---
docs/en/docs/admin-manual/config/fe-config.md | 9 +++++++
.../docs/ecosystem/external-table/jdbc-of-doris.md | 2 +-
docs/zh-CN/docs/admin-manual/config/fe-config.md | 17 +++++++++++++
.../docs/ecosystem/external-table/jdbc-of-doris.md | 2 +-
.../main/java/org/apache/doris/common/Config.java | 7 ++++++
.../org/apache/doris/planner/JdbcScanNode.java | 2 +-
.../org/apache/doris/planner/OdbcScanNode.java | 29 ++++++++++++++++------
7 files changed, 57 insertions(+), 11 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index 339e4d836b..2f01c58a50 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2358,3 +2358,12 @@ Is it possible to dynamically configure: true
Is it a configuration item unique to the Master FE node: true
+#### `enable_func_pushdown`
+
+Default：true
+
+IsMutable：true
+
+MasterOnly：false
+
+Whether to push the filter conditions with functions down to MYSQL, when exectue query of ODBC、JDBC external tables
diff --git a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md
index 8d19f7ffab..8b79077251 100644
--- a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md
+++ b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md
@@ -198,7 +198,7 @@ There are different data types among different databases. Here is a list of the
| DATE | DATETIME |
| SMALLINT | SMALLINT |
| INT | INT |
-| REAL | FLOAT |
+| REAL | DOUBLE |
| FLOAT | DOUBLE |
| NUMBER | DECIMAL |
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index b7cb03e9f8..d6488e2995 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2412,3 +2412,20 @@ hive partition 的最大缓存数量。
是否为 Master FE 节点独有的配置项：true
+#### `proxy_auth_magic_prefix`
+
+默认值：x@8
+
+#### `proxy_auth_enable`
+
+默认值：false
+
+#### `enable_func_pushdown`
+
+默认值：true
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：false
+
+在ODBC、JDBC的MYSQL外部表查询时，是否将带函数的过滤条件下推到MYSQL中执行
diff --git a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md
index 4ef47e1d66..eef14642e3 100644
--- a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md
+++ b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md
@@ -201,7 +201,7 @@ PROPERTIES (
| DATE | DATETIME |
| SMALLINT | SMALLINT |
| INT | INT |
-| REAL | FLOAT |
+| REAL | DOUBLE |
| FLOAT | DOUBLE |
| NUMBER | DECIMAL |
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index 484ed5c7e9..de5b4fac41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1926,5 +1926,12 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = false)
public static boolean disable_enable_vectorized_engine = true;
+
+ /**
+ * This is used whether to push down function to MYSQL in external Table with query sql
+ * like odbc, jdbc for mysql table
+ */
+ @ConfField(mutable = true)
+ public static boolean enable_func_pushdown = true;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java
index b06956fda3..b9da4b45c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java
@@ -102,7 +102,7 @@ public class JdbcScanNode extends ScanNode {
ArrayList<Expr> conjunctsList = Expr.cloneList(conjuncts, sMap);
for (Expr p : conjunctsList) {
if (OdbcScanNode.shouldPushDownConjunct(jdbcType, p)) {
- String filter = p.toMySql();
+ String filter = OdbcScanNode.conjunctExprToString(jdbcType, p);
filters.add(filter);
conjuncts.remove(p);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
index bfec4e871f..f47be29d38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
@@ -18,6 +18,7 @@
package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprSubstitutionMap;
@@ -27,6 +28,8 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OdbcTable;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
@@ -54,7 +57,6 @@ public class OdbcScanNode extends ScanNode {
// Now some database have different function call like doris, now doris do not
// push down the function call except MYSQL
- // TODO: maybe add a config to decide whether to pushdown the function of MYSQL
public static boolean shouldPushDownConjunct(TOdbcTableType tableType, Expr expr) {
if (!tableType.equals(TOdbcTableType.MYSQL)) {
List<FunctionCallExpr> fnExprList = Lists.newArrayList();
@@ -62,14 +64,25 @@ public class OdbcScanNode extends ScanNode {
if (!fnExprList.isEmpty()) {
return false;
}
- //oracle date type is not push down https://github.com/apache/doris/discussions/15069
- //TODO: Now not sure how to rewrite the needed expr and doesn't affect other normal
- //if we could rewrite the expr according to the format, then can pushdown this.
- if (tableType.equals(TOdbcTableType.ORACLE) && expr.contains(DateLiteral.class)) {
- return false;
+ }
+ return Config.enable_func_pushdown;
+ }
+
+ public static String conjunctExprToString(TOdbcTableType tableType, Expr expr) {
+ if (tableType.equals(TOdbcTableType.ORACLE) && expr.contains(DateLiteral.class)
+ && (expr instanceof BinaryPredicate)) {
+ ArrayList<Expr> children = expr.getChildren();
+ // k1 OP '2022-12-10 20:55:59' changTo ---> k1 OP to_date('{}','yyyy-mm-dd hh24:mi:ss')
+ // oracle datetime push down is different: https://github.com/apache/doris/discussions/15069
+ if (children.get(1).isConstant() && (children.get(1).getType().equals(Type.DATETIME) || children
+ .get(1).getType().equals(Type.DATETIMEV2))) {
+ String filter = children.get(0).toSql();
+ filter += ((BinaryPredicate) expr).getOp().toString();
+ filter += "to_date('" + children.get(1).getStringValue() + "','yyyy-mm-dd hh24:mi:ss')";
+ return filter;
}
}
- return true;
+ return expr.toMySql();
}
private final List<String> columns = new ArrayList<String>();
@@ -190,7 +203,7 @@ public class OdbcScanNode extends ScanNode {
ArrayList<Expr> odbcConjuncts = Expr.cloneList(conjuncts, sMap);
for (Expr p : odbcConjuncts) {
if (shouldPushDownConjunct(odbcType, p)) {
- String filter = p.toMySql();
+ String filter = conjunctExprToString(odbcType, p);
filters.add(filter);
conjuncts.remove(p);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org