You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/31 04:55:40 UTC
[doris] branch branch-1.2-lts updated (afda226d41 -> 389e323289)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from afda226d41 [fix](merge-on-write) unique key mow tables should require distribution columns be key column (#15537)
new 11708482a6 [Fix](SparkLoad): fix the timeout aborted loadtasks are not cleaned up. (#15480)
new 389e323289 [vectorized](jdbc) fix external table of oracle have keyworld column (#15487)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/en/docs/ecosystem/external-table/jdbc-of-doris.md | 2 ++
docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md | 1 +
fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java | 7 +++++++
.../src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java | 9 +++++++++
.../src/test/java/org/apache/doris/planner/QueryPlanTest.java | 4 ++--
5 files changed, 21 insertions(+), 2 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [Fix](SparkLoad): fix the timeout aborted loadtasks are not cleaned up. (#15480)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 11708482a68a33af2b9117270c0c77ce9351f2f6
Author: spaces-x <we...@gmail.com>
AuthorDate: Fri Dec 30 14:02:00 2022 +0800
[Fix](SparkLoad): fix the timeout aborted loadtasks are not cleaned up. (#15480)
Co-authored-by: spaces-x <we...@meituan.com>
---
.../src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
index 83fe2c7d64..72bad582d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
@@ -695,6 +695,8 @@ public class SparkLoadJob extends BulkLoadJob {
// clear job infos that not persist
sparkLoadAppHandle = null;
resourceDesc = null;
+ etlOutputPath = "";
+ appId = "";
tableToLoadPartitions.clear();
indexToPushBrokerReaderParams.clear();
indexToSchemaHash.clear();
@@ -713,6 +715,13 @@ public class SparkLoadJob extends BulkLoadJob {
clearJob();
}
+ @Override
+ public void afterAborted(TransactionState txnState, boolean txnOperated, String txnStatusChangeReason)
+ throws UserException {
+ super.afterAborted(txnState, txnOperated, txnStatusChangeReason);
+ clearJob();
+ }
+
@Override
public void cancelJobWithoutCheck(FailMsg failMsg, boolean abortTxn, boolean needLog) {
super.cancelJobWithoutCheck(failMsg, abortTxn, needLog);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: [vectorized](jdbc) fix external table of oracle have keyworld column (#15487)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 389e323289cb877383098aaa962f7120f1eea1ab
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Sat Dec 31 12:48:26 2022 +0800
[vectorized](jdbc) fix external table of oracle have keyworld column (#15487)
if column name is keyword of oracle, the query will report error
---
docs/en/docs/ecosystem/external-table/jdbc-of-doris.md | 2 ++
docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md | 1 +
fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java | 7 +++++++
.../src/test/java/org/apache/doris/planner/QueryPlanTest.java | 4 ++--
4 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md
index d2479131bd..74e0c368b3 100644
--- a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md
+++ b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md
@@ -101,6 +101,8 @@ Parameter Description：
```
select * from mysql_table where k1 > 1000 and k3 ='term';
```
+Because it is possible to use keywords in the database as column name, in order to solve this problem, escape characters will be automatically added to field names and table names in SQL statements according to the standards of each database. For example, MYSQL (``), PostgreSQL (""), SQLServer ([]), and ORACLE (""), But this may cause case sensitivity of field names. You can check the query statements issued to each database after escape through explain SQL.
+
### Data write
After the JDBC external table is create in Doris, the data can be written directly by the `insert into` statement, the query results of Doris can be written to the JDBC external table, or the data can be imported from one JDBC table to another.
diff --git a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md
index 68e63682aa..b60a3f5ec4 100644
--- a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md
+++ b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md
@@ -87,6 +87,7 @@ PROPERTIES (
```
select * from mysql_table where k1 > 1000 and k3 ='term';
```
+由于可能存在使用数据库内部的关键字作为字段名，为解决这种状况下仍能正确查询，所以在SQL语句中，会根据各个数据库的标准自动在字段名与表名上加上转义符。例如 MYSQL(``)、PostgreSQL("")、SQLServer([])、ORACLE("")，所以此时可能会造成字段名的大小写敏感，具体可以通过explain sql，查看转义后下发到各个数据库的查询语句。
### 数据写入
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java
index b2464c257f..562f4d7807 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java
@@ -96,6 +96,11 @@ public class OdbcTable extends Table {
return list.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining("."));
}
+ private static String oracleProperName(String name) {
+ List<String> list = Arrays.asList(name.split("\\."));
+ return list.stream().map(s -> "\"" + s.toUpperCase() + "\"").collect(Collectors.joining("."));
+ }
+
public static String databaseProperName(TOdbcTableType tableType, String name) {
switch (tableType) {
case MYSQL:
@@ -104,6 +109,8 @@ public class OdbcTable extends Table {
return mssqlProperName(name);
case POSTGRESQL:
return psqlProperName(name);
+ case ORACLE:
+ return oracleProperName(name);
default:
return name;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index 914bc338bc..4f41e70c45 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -1312,7 +1312,7 @@ public class QueryPlanTest extends TestWithFeService {
// this table is Oracle ODBC table, so abs(k1) should not be pushed down
queryStr = "explain select * from odbc_oracle where k1 > 10 and abs(k1) > 10";
explainString = getSQLPlanOrErrorMsg(queryStr);
- Assert.assertTrue(explainString.contains("k1 > 10"));
+ Assert.assertTrue(explainString.contains("\"K1\" > 10"));
Assert.assertTrue(!explainString.contains("abs(k1) > 10"));
}
@@ -1352,7 +1352,7 @@ public class QueryPlanTest extends TestWithFeService {
String explainString = getSQLPlanOrErrorMsg(queryStr);
Assert.assertTrue(explainString.contains("TABLENAME IN DORIS: odbc_oracle"));
Assert.assertTrue(explainString.contains("TABLE TYPE: ORACLE"));
- Assert.assertTrue(explainString.contains("TABLENAME OF EXTERNAL TABLE: tbl1"));
+ Assert.assertTrue(explainString.contains("TABLENAME OF EXTERNAL TABLE: \"TBL1\""));
// enable transaction of ODBC Sink
Deencapsulation.setField(connectContext.getSessionVariable(), "enableOdbcTransaction", true);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org