You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/12/17 14:16:54 UTC
[12/21] carbondata git commit: [CARBONDATA-2563][CATALYST] Explain
query with Order by operator is fired Spark Job which is increase explain
query time even though isQueryStatisticsEnabled is false
[CARBONDATA-2563][CATALYST] Explain query with Order by operator is fired Spark Job
which is increase explain query time even though isQueryStatisticsEnabled is false
Even though isQueryStatisticsEnabled is false which means user doesnt wants to see
statistics for explain command, still the engine tries to fetch the paritions information
which causes a job execution in case of order by query, this is mainly because
spark engine does sampling for defifning certain range within paritions for sorting process.
As part of solution the explain command process shall fetch the parition info only if isQueryStatisticsEnabled true.
This closes #2974
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/21762c54
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/21762c54
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/21762c54
Branch: refs/heads/branch-1.5
Commit: 21762c541df334816c660eb8e4e26aca1df9aa71
Parents: 93268ea
Author: s71955 <su...@gmail.com>
Authored: Tue Dec 4 20:48:55 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/profiler/ExplainCollector.java | 6 +++++-
.../execution/command/table/CarbonExplainCommand.scala | 11 ++++++++---
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21762c54/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
index 8513dac..75568e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
@@ -171,7 +171,11 @@ public class ExplainCollector {
}
public static String getFormatedOutput() {
- return get().toString();
+ if (null != get()) {
+ return get().toString();
+ } else {
+ return null;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21762c54/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
index 8939c6a..aa7a541 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
@@ -47,12 +47,17 @@ case class CarbonExplainCommand(
}
private def collectProfiler(sparkSession: SparkSession): Seq[Row] = {
- val queryExecution =
- sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan)
try {
ExplainCollector.setup()
- queryExecution.toRdd.partitions
if (ExplainCollector.enabled()) {
+ val queryExecution =
+ sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan)
+ queryExecution.toRdd.partitions
+ // For count(*) queries the explain collector will be disabled, so profiler
+ // informations not required in such scenarios.
+ if (null == ExplainCollector.getFormatedOutput) {
+ Seq.empty
+ }
Seq(Row("== CarbonData Profiler ==\n" + ExplainCollector.getFormatedOutput))
} else {
Seq.empty