You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/04/08 06:25:12 UTC
carbondata git commit: [CARBONDATA-2319][Profiler] Fix
carbon_scan_time and carbon_IO_time in task statistics
Repository: carbondata
Updated Branches:
refs/heads/master 85a958ee3 -> f6990d622
[CARBONDATA-2319][Profiler] Fix carbon_scan_time and carbon_IO_time in task statistics
carbon_scan_time and carbon_IO_time are incorrect in task statistics. This PR fix it
This closes #2144
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f6990d62
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f6990d62
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f6990d62
Branch: refs/heads/master
Commit: f6990d62293d7f1fc0ab449e812aeca0d7e1d130
Parents: 85a958e
Author: QiangCai <qi...@qq.com>
Authored: Sun Apr 8 11:01:00 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Apr 8 14:25:01 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/stats/TaskStatistics.java | 4 ++--
.../carbondata/examples/CarbonSessionExample.scala | 16 +++++++++-------
2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f6990d62/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java b/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
index 7e4d89f..25e5542 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
@@ -37,8 +37,8 @@ public class TaskStatistics implements Serializable {
new Column("total_time", QueryStatisticsConstants.EXECUTOR_PART, true),
new Column("load_blocks_time", QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, true),
new Column("load_dictionary_time", QueryStatisticsConstants.LOAD_DICTIONARY, true),
- new Column("carbon_scan_time", QueryStatisticsConstants.SCAN_BLOCKlET_TIME, true),
- new Column("carbon_IO_time", QueryStatisticsConstants.READ_BLOCKlET_TIME, true),
+ new Column("carbon_scan_time", QueryStatisticsConstants.SCAN_BLOCKlET_TIME),
+ new Column("carbon_IO_time", QueryStatisticsConstants.READ_BLOCKlET_TIME),
new Column("scan_blocks_num", QueryStatisticsConstants.SCAN_BLOCKS_NUM),
new Column("total_blocklets", QueryStatisticsConstants.TOTAL_BLOCKLET_NUM),
new Column("valid_blocklets", QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f6990d62/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index d254152..a0ceeca 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -29,6 +29,15 @@ import org.apache.carbondata.examples.util.ExampleUtils
object CarbonSessionExample {
def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ System.setProperty("path.target", s"$rootPath/examples/spark2/target")
+ // print profiler log to a separated file: target/profiler.log
+ PropertyConfigurator.configure(
+ s"$rootPath/examples/spark2/src/main/resources/log4j.properties")
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true")
val spark = ExampleUtils.createCarbonSession("CarbonSessionExample")
spark.sparkContext.setLogLevel("INFO")
exampleBody(spark)
@@ -39,13 +48,6 @@ object CarbonSessionExample {
val rootPath = new File(this.getClass.getResource("/").getPath
+ "../../../..").getCanonicalPath
- System.setProperty("path.target", s"$rootPath/examples/spark2/target")
- // print profiler log to a separated file: target/profiler.log
- PropertyConfigurator.configure(
- s"$rootPath/examples/spark2/src/main/resources/log4j.properties")
-
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true")
spark.sql("DROP TABLE IF EXISTS carbonsession_table")