You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/11/18 06:01:58 UTC
[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4822 The metrics
'Total spark scan time' of query log is negative in some cases
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new 0b4df4c KYLIN-4822 The metrics 'Total spark scan time' of query log is negative in some cases
0b4df4c is described below
commit 0b4df4cf67b27c6f9a7a9e5c9006ddf528acc09b
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Wed Nov 18 09:08:46 2020 +0800
KYLIN-4822 The metrics 'Total spark scan time' of query log is negative in some cases
---
.../apache/spark/sql/hive/utils/QueryMetricUtils.scala | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/hive/utils/QueryMetricUtils.scala b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/hive/utils/QueryMetricUtils.scala
index d4f8c50..c928e84 100644
--- a/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/hive/utils/QueryMetricUtils.scala
+++ b/kylin-spark-project/kylin-spark-common/src/main/scala/org/apache/spark/sql/hive/utils/QueryMetricUtils.scala
@@ -42,11 +42,16 @@ object QueryMetricUtils extends Logging {
// There is only 'numOutputRows' metric in HiveTableScanExec
(exec.metrics.apply("numOutputRows").value, -1l, -1l, -1l, -1l)
}
- val scanRows = metrics.map(metric => java.lang.Long.valueOf(metric._1)).toList.asJava
- val scanFiles = metrics.map(metrics => java.lang.Long.valueOf(metrics._2)).toList.asJava
- val metadataTime = metrics.map(metrics => java.lang.Long.valueOf(metrics._3)).toList.asJava
- val scanTime = metrics.map(metrics => java.lang.Long.valueOf(metrics._4)).toList.asJava
- val scanBytes = metrics.map(metric => java.lang.Long.valueOf(metric._5)).toList.asJava
+ val scanRows = metrics.map(metric => java.lang.Long.valueOf(metric._1))
+ .filter(_ >= 0L).toList.asJava
+ val scanFiles = metrics.map(metrics => java.lang.Long.valueOf(metrics._2))
+ .filter(_ >= 0L).toList.asJava
+ val metadataTime = metrics.map(metrics => java.lang.Long.valueOf(metrics._3))
+ .filter(_ >= 0L).toList.asJava
+ val scanTime = metrics.map(metrics => java.lang.Long.valueOf(metrics._4))
+ .filter(_ >= 0L).toList.asJava
+ val scanBytes = metrics.map(metric => java.lang.Long.valueOf(metric._5))
+ .filter(_ >= 0L).toList.asJava
(scanRows, scanFiles, metadataTime, scanTime, scanBytes)
} catch {