You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/12/13 05:45:52 UTC
carbondata git commit: [CARBONDATA-2563][CATALYST] Explain query with
Order by operator is fired Spark Job which is increase explain query time
even though isQueryStatisticsEnabled is false
Repository: carbondata
Updated Branches:
refs/heads/master 6026cb57c -> d3c907b71
[CARBONDATA-2563][CATALYST] Explain query with Order by operator is fired Spark Job
which is increase explain query time even though isQueryStatisticsEnabled is false
Even though isQueryStatisticsEnabled is false which means user doesnt wants to see
statistics for explain command, still the engine tries to fetch the paritions information
which causes a job execution in case of order by query, this is mainly because
spark engine does sampling for defifning certain range within paritions for sorting process.
As part of solution the explain command process shall fetch the parition info only if isQueryStatisticsEnabled true.
This closes #2974
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d3c907b7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d3c907b7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d3c907b7
Branch: refs/heads/master
Commit: d3c907b71a6dbc5fb9b2c0167959de7763d1032d
Parents: 6026cb5
Author: s71955 <su...@gmail.com>
Authored: Tue Dec 4 20:48:55 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Thu Dec 13 11:14:31 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/profiler/ExplainCollector.java | 6 +++++-
.../execution/command/table/CarbonExplainCommand.scala | 11 ++++++++---
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3c907b7/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
index 8513dac..75568e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
@@ -171,7 +171,11 @@ public class ExplainCollector {
}
public static String getFormatedOutput() {
- return get().toString();
+ if (null != get()) {
+ return get().toString();
+ } else {
+ return null;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d3c907b7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
index 8939c6a..aa7a541 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
@@ -47,12 +47,17 @@ case class CarbonExplainCommand(
}
private def collectProfiler(sparkSession: SparkSession): Seq[Row] = {
- val queryExecution =
- sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan)
try {
ExplainCollector.setup()
- queryExecution.toRdd.partitions
if (ExplainCollector.enabled()) {
+ val queryExecution =
+ sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan)
+ queryExecution.toRdd.partitions
+ // For count(*) queries the explain collector will be disabled, so profiler
+ // informations not required in such scenarios.
+ if (null == ExplainCollector.getFormatedOutput) {
+ Seq.empty
+ }
Seq(Row("== CarbonData Profiler ==\n" + ExplainCollector.getFormatedOutput))
} else {
Seq.empty