You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2017/08/21 12:20:45 UTC
spark git commit: [SPARK-21718][SQL] Heavy log of type: "Skipping
partition based on stats ..."
Repository: spark
Updated Branches:
refs/heads/master 77d046ec4 -> b3a07526f
[SPARK-21718][SQL] Heavy log of type: "Skipping partition based on stats ..."
## What changes were proposed in this pull request?
Reduce 'Skipping partitions' message to debug
## How was this patch tested?
Existing tests
Author: Sean Owen <so...@cloudera.com>
Closes #19010 from srowen/SPARK-21718.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b3a07526
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b3a07526
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b3a07526
Branch: refs/heads/master
Commit: b3a07526fe774fd64fe3a2b9a2381eff9a3c49a3
Parents: 77d046e
Author: Sean Owen <so...@cloudera.com>
Authored: Mon Aug 21 14:20:40 2017 +0200
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Mon Aug 21 14:20:40 2017 +0200
----------------------------------------------------------------------
.../sql/execution/columnar/InMemoryTableScanExec.scala | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b3a07526/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
index 1d60137..c7ddec5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
@@ -166,12 +166,13 @@ case class InMemoryTableScanExec(
if (inMemoryPartitionPruningEnabled) {
cachedBatchIterator.filter { cachedBatch =>
if (!partitionFilter.eval(cachedBatch.stats)) {
- def statsString: String = schemaIndex.map {
- case (a, i) =>
+ logDebug {
+ val statsString = schemaIndex.map { case (a, i) =>
val value = cachedBatch.stats.get(i, a.dataType)
s"${a.name}: $value"
- }.mkString(", ")
- logInfo(s"Skipping partition based on stats $statsString")
+ }.mkString(", ")
+ s"Skipping partition based on stats $statsString"
+ }
false
} else {
true
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org