You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/03/02 02:17:31 UTC
[incubator-doris] branch master updated: [fix](fold-constant)(hive) fix constant-folding in order by and optimize logs in hive client (#8268)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f41316a [fix](fold-constant)(hive) fix constant-folding in order by and optimize logs in hive client (#8268)
f41316a is described below
commit f41316a3ec4015b317006078978f4c860c5d8832
Author: qiye <ji...@gmail.com>
AuthorDate: Wed Mar 2 10:17:17 2022 +0800
[fix](fold-constant)(hive) fix constant-folding in order by and optimize logs in hive client (#8268)
fix: #7509
1. fix order by clause constant folding
2. optimize Hive exception message
3. change hive file status log type
---
.../java/org/apache/doris/analysis/SelectStmt.java | 7 ++++---
.../doris/catalog/HiveMetaStoreClientHelper.java | 20 ++++++++++----------
.../java/org/apache/doris/planner/HiveScanNode.java | 2 +-
3 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 502a7fd..779800c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -1450,7 +1450,7 @@ public class SelectStmt extends QueryStmt {
* ORDER BY `b` DESC
* ```
* Aliases information of groupBy and orderBy clauses is recorded in `QueryStmt.aliasSMap`.
- * The select clause has it's own alias info in `SelectListItem.alias`.
+ * The select clause has its own alias info in `SelectListItem.alias`.
*
* Aliases expr in the `group by` and `order by` clauses are not analyzed, i.e. `Expr.isAnalyzed=false`
* Subsequent constant folding will analyze the unanalyzed Expr before collecting the constant
@@ -1471,7 +1471,7 @@ public class SelectStmt extends QueryStmt {
}
}
if (orderByElements != null) {
- for (OrderByElement orderByElem : orderByElements) {
+ for (OrderByElement orderByElem : orderByElementsAfterAnalyzed) {
// same as above
if (containAlias(orderByElem.getExpr())) {
continue;
@@ -1548,7 +1548,7 @@ public class SelectStmt extends QueryStmt {
}
}
if (orderByElements != null) {
- for (OrderByElement orderByElem : orderByElements) {
+ for (OrderByElement orderByElem : orderByElementsAfterAnalyzed) {
Expr expr = orderByElem.getExpr();
if (expr.getId() == null) {
orderByElem.setExpr(expr);
@@ -1556,6 +1556,7 @@ public class SelectStmt extends QueryStmt {
orderByElem.setExpr(rewrittenExprMap.get(expr.getId().toString()));
}
}
+ orderByElements = (ArrayList<OrderByElement>) orderByElementsAfterAnalyzed;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 5dcfd12..8ca9d7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -126,7 +126,7 @@ public class HiveMetaStoreClientHelper {
hivemetastoreclient = new HiveMetaStoreClient(hiveConf);
} catch (MetaException e) {
LOG.warn("Create HiveMetaStoreClient failed: {}", e.getMessage());
- throw new DdlException("Create HiveMetaStoreClient failed");
+ throw new DdlException("Create HiveMetaStoreClient failed: " + e.getMessage());
}
return hivemetastoreclient;
}
@@ -144,7 +144,7 @@ public class HiveMetaStoreClientHelper {
return client.tableExists(dbName, tblName);
} catch (TException e) {
LOG.warn("Hive metastore thrift exception: {}", e.getMessage());
- throw new DdlException("Connect hive metastore failed.");
+ throw new DdlException("Connect hive metastore failed. Error: " + e.getMessage());
} finally {
dropClient(client);
}
@@ -179,7 +179,7 @@ public class HiveMetaStoreClientHelper {
SerializationUtilities.serializeExpressionToKryo(hivePartitionPredicate), null, (short) -1, hivePartitions);
} catch (TException e) {
LOG.warn("Hive metastore thrift exception: {}", e.getMessage());
- throw new DdlException("Connect hive metastore failed.");
+ throw new DdlException("Connect hive metastore failed. Error: " + e.getMessage());
} finally {
client.close();
}
@@ -212,7 +212,7 @@ public class HiveMetaStoreClientHelper {
}
} catch (IOException e) {
LOG.warn("List HDFS file IOException: {}", e.getMessage());
- throw new DdlException("List HDFS file failed.");
+ throw new DdlException("List HDFS file failed. Error: " + e.getMessage());
}
}
@@ -230,7 +230,7 @@ public class HiveMetaStoreClientHelper {
iterators.add(fileSystem.listLocatedStatus(path));
} catch (IOException e) {
LOG.warn("Get HDFS file remote iterator failed. {}", e.getMessage());
- throw new DdlException("Get HDFS file remote iterator failed.");
+ throw new DdlException("Get HDFS file remote iterator failed. Error: " + e.getMessage());
}
}
return iterators;
@@ -246,7 +246,7 @@ public class HiveMetaStoreClientHelper {
iterators.add(fileSystem.listLocatedStatus(path));
} catch (IOException e) {
LOG.warn("Get HDFS file remote iterator failed. {}" + e.getMessage());
- throw new DdlException("Get HDFS file remote iterator failed.");
+ throw new DdlException("Get HDFS file remote iterator failed. Error: " + e.getMessage());
}
return iterators;
}
@@ -258,7 +258,7 @@ public class HiveMetaStoreClientHelper {
partitionNames = client.listPartitionNames(hiveTable.getHiveDb(), hiveTable.getHiveTable(), (short) -1);
} catch (TException e) {
LOG.warn("Hive metastore thrift exception: {}", e.getMessage());
- throw new DdlException("Connect hive metastore failed.");
+ throw new DdlException("Connect hive metastore failed. Error: " + e.getMessage());
}
return partitionNames;
@@ -271,7 +271,7 @@ public class HiveMetaStoreClientHelper {
table = client.getTable(hiveTable.getHiveDb(), hiveTable.getHiveTable());
} catch (TException e) {
LOG.warn("Hive metastore thrift exception: {}", e.getMessage());
- throw new DdlException("Connect hive metastore failed.");
+ throw new DdlException("Connect hive metastore failed. Error: " + e.getMessage());
}
return table;
}
@@ -409,7 +409,7 @@ public class HiveMetaStoreClientHelper {
FunctionRegistry.getFunctionInfo(op).getGenericUDF(), args);
} catch (SemanticException e) {
LOG.warn("Convert to Hive expr failed: {}", e.getMessage());
- throw new DdlException("Convert to Hive expr failed.");
+ throw new DdlException("Convert to Hive expr failed. Error: " + e.getMessage());
}
return compoundExpr;
}
@@ -534,7 +534,7 @@ public class HiveMetaStoreClientHelper {
FunctionRegistry.getFunctionInfo(name).getGenericUDF(), children));
} catch (SemanticException e) {
LOG.warn("Build Hive expression failed: semantic analyze exception: {}", e.getMessage());
- throw new DdlException("Build Hive expression Failed");
+ throw new DdlException("Build Hive expression Failed. Error: " + e.getMessage());
}
return this;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
index a67f539..76cf534 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
@@ -183,7 +183,7 @@ public class HiveScanNode extends BrokerScanNode {
fileStatusesList.add(fileStatuses);
filesAdded += fileStatuses.size();
for (TBrokerFileStatus fstatus : fileStatuses) {
- LOG.info("Add file status is {}", fstatus);
+ LOG.debug("Add file status is {}", fstatus);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org