You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2018/11/21 19:17:01 UTC
spark git commit: [SPARK-26066][SQL] Move truncatedString to
sql/catalyst and add spark.sql.debug.maxToStringFields conf
Repository: spark
Updated Branches:
refs/heads/master 07a700b37 -> 81550b38e
[SPARK-26066][SQL] Move truncatedString to sql/catalyst and add spark.sql.debug.maxToStringFields conf
## What changes were proposed in this pull request?
In the PR, I propose:
- new SQL config `spark.sql.debug.maxToStringFields` to control maximum number fields up to which `truncatedString` cuts its input sequences.
- Moving `truncatedString` out of `core` to `sql/catalyst` because it is used only in the `sql/catalyst` packages for restricting number of fields converted to strings from `TreeNode` and expressions of`StructType`.
## How was this patch tested?
Added a test to `QueryExecutionSuite` to check that `spark.sql.debug.maxToStringFields` impacts to behavior of `truncatedString`.
Closes #23039 from MaxGekk/truncated-string-catalyst.
Lead-authored-by: Maxim Gekk <ma...@databricks.com>
Co-authored-by: Maxim Gekk <ma...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/81550b38
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/81550b38
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/81550b38
Branch: refs/heads/master
Commit: 81550b38e43fb20f89f529d2127575c71a54a538
Parents: 07a700b
Author: Maxim Gekk <ma...@databricks.com>
Authored: Wed Nov 21 11:16:54 2018 -0800
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Wed Nov 21 11:16:54 2018 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/util/Utils.scala | 48 --------------------
.../org/apache/spark/util/UtilsSuite.scala | 8 ----
.../sql/catalyst/expressions/Expression.scala | 4 +-
.../plans/logical/basicLogicalOperators.scala | 4 +-
.../spark/sql/catalyst/trees/TreeNode.scala | 10 ++--
.../spark/sql/catalyst/util/package.scala | 37 ++++++++++++++-
.../org/apache/spark/sql/internal/SQLConf.scala | 9 ++++
.../org/apache/spark/sql/types/StructType.scala | 4 +-
.../org/apache/spark/sql/util/UtilSuite.scala | 31 +++++++++++++
.../sql/execution/DataSourceScanExec.scala | 5 +-
.../spark/sql/execution/ExistingRDD.scala | 4 +-
.../spark/sql/execution/QueryExecution.scala | 3 +-
.../execution/aggregate/HashAggregateExec.scala | 7 +--
.../aggregate/ObjectHashAggregateExec.scala | 8 ++--
.../execution/aggregate/SortAggregateExec.scala | 8 ++--
.../execution/columnar/InMemoryRelation.scala | 5 +-
.../execution/datasources/LogicalRelation.scala | 4 +-
.../datasources/jdbc/JDBCRelation.scala | 5 +-
.../v2/DataSourceV2StringFormat.scala | 5 +-
.../org/apache/spark/sql/execution/limit.scala | 6 +--
.../streaming/MicroBatchExecution.scala | 7 +--
.../continuous/ContinuousExecution.scala | 7 +--
.../spark/sql/execution/streaming/memory.scala | 4 +-
.../sql/execution/QueryExecutionSuite.scala | 26 +++++++++++
24 files changed, 156 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 743fd5d..227c9e7 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -31,7 +31,6 @@ import java.security.SecureRandom
import java.util.{Locale, Properties, Random, UUID}
import java.util.concurrent._
import java.util.concurrent.TimeUnit.NANOSECONDS
-import java.util.concurrent.atomic.AtomicBoolean
import java.util.zip.GZIPInputStream
import scala.annotation.tailrec
@@ -93,53 +92,6 @@ private[spark] object Utils extends Logging {
private val MAX_DIR_CREATION_ATTEMPTS: Int = 10
@volatile private var localRootDirs: Array[String] = null
- /**
- * The performance overhead of creating and logging strings for wide schemas can be large. To
- * limit the impact, we bound the number of fields to include by default. This can be overridden
- * by setting the 'spark.debug.maxToStringFields' conf in SparkEnv.
- */
- val DEFAULT_MAX_TO_STRING_FIELDS = 25
-
- private[spark] def maxNumToStringFields = {
- if (SparkEnv.get != null) {
- SparkEnv.get.conf.getInt("spark.debug.maxToStringFields", DEFAULT_MAX_TO_STRING_FIELDS)
- } else {
- DEFAULT_MAX_TO_STRING_FIELDS
- }
- }
-
- /** Whether we have warned about plan string truncation yet. */
- private val truncationWarningPrinted = new AtomicBoolean(false)
-
- /**
- * Format a sequence with semantics similar to calling .mkString(). Any elements beyond
- * maxNumToStringFields will be dropped and replaced by a "... N more fields" placeholder.
- *
- * @return the trimmed and formatted string.
- */
- def truncatedString[T](
- seq: Seq[T],
- start: String,
- sep: String,
- end: String,
- maxNumFields: Int = maxNumToStringFields): String = {
- if (seq.length > maxNumFields) {
- if (truncationWarningPrinted.compareAndSet(false, true)) {
- logWarning(
- "Truncated the string representation of a plan since it was too large. This " +
- "behavior can be adjusted by setting 'spark.debug.maxToStringFields' in SparkEnv.conf.")
- }
- val numFields = math.max(0, maxNumFields - 1)
- seq.take(numFields).mkString(
- start, sep, sep + "... " + (seq.length - numFields) + " more fields" + end)
- } else {
- seq.mkString(start, sep, end)
- }
- }
-
- /** Shorthand for calling truncatedString() without start or end strings. */
- def truncatedString[T](seq: Seq[T], sep: String): String = truncatedString(seq, "", sep, "")
-
/** Serialize an object using Java serialization */
def serialize[T](o: T): Array[Byte] = {
val bos = new ByteArrayOutputStream()
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 5293645..f5e912b 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -45,14 +45,6 @@ import org.apache.spark.scheduler.SparkListener
class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
- test("truncatedString") {
- assert(Utils.truncatedString(Nil, "[", ", ", "]", 2) == "[]")
- assert(Utils.truncatedString(Seq(1, 2), "[", ", ", "]", 2) == "[1, 2]")
- assert(Utils.truncatedString(Seq(1, 2, 3), "[", ", ", "]", 2) == "[1, ... 2 more fields]")
- assert(Utils.truncatedString(Seq(1, 2, 3), "[", ", ", "]", -5) == "[, ... 3 more fields]")
- assert(Utils.truncatedString(Seq(1, 2, 3), ", ") == "1, 2, 3")
- }
-
test("timeConversion") {
// Test -1
assert(Utils.timeStringAsSeconds("-1") === -1)
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 141fcff..d51b110 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -25,9 +25,9 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
////////////////////////////////////////////////////////////////////////////////////////////////////
// This file defines the basic expression abstract classes in Catalyst.
@@ -237,7 +237,7 @@ abstract class Expression extends TreeNode[Expression] {
override def simpleString: String = toString
- override def toString: String = prettyName + Utils.truncatedString(
+ override def toString: String = prettyName + truncatedString(
flatArguments.toSeq, "(", ", ", ")")
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index f09c5ce..07fa17b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -24,8 +24,8 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
import org.apache.spark.util.random.RandomSampler
/**
@@ -485,7 +485,7 @@ case class With(child: LogicalPlan, cteRelations: Seq[(String, SubqueryAlias)])
override def output: Seq[Attribute] = child.output
override def simpleString: String = {
- val cteAliases = Utils.truncatedString(cteRelations.map(_._1), "[", ", ", "]")
+ val cteAliases = truncatedString(cteRelations.map(_._1), "[", ", ", "]")
s"CTE $cteAliases"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index 1027216..2e9f9f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -37,9 +37,9 @@ import org.apache.spark.sql.catalyst.errors._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.JoinType
import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, Partitioning}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
-import org.apache.spark.util.Utils
/** Used by [[TreeNode.getNodeNumbered]] when traversing the tree for a given number */
private class MutableInt(var i: Int)
@@ -440,10 +440,10 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
case tn: TreeNode[_] => tn.simpleString :: Nil
case seq: Seq[Any] if seq.toSet.subsetOf(allChildren.asInstanceOf[Set[Any]]) => Nil
case iter: Iterable[_] if iter.isEmpty => Nil
- case seq: Seq[_] => Utils.truncatedString(seq, "[", ", ", "]") :: Nil
- case set: Set[_] => Utils.truncatedString(set.toSeq, "{", ", ", "}") :: Nil
+ case seq: Seq[_] => truncatedString(seq, "[", ", ", "]") :: Nil
+ case set: Set[_] => truncatedString(set.toSeq, "{", ", ", "}") :: Nil
case array: Array[_] if array.isEmpty => Nil
- case array: Array[_] => Utils.truncatedString(array, "[", ", ", "]") :: Nil
+ case array: Array[_] => truncatedString(array, "[", ", ", "]") :: Nil
case null => Nil
case None => Nil
case Some(null) => Nil
@@ -664,7 +664,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
t.forall(_.isInstanceOf[Partitioning]) || t.forall(_.isInstanceOf[DataType]) =>
JArray(t.map(parseToJson).toList)
case t: Seq[_] if t.length > 0 && t.head.isInstanceOf[String] =>
- JString(Utils.truncatedString(t, "[", ", ", "]"))
+ JString(truncatedString(t, "[", ", ", "]"))
case t: Seq[_] => JNull
case m: Map[_, _] => JNull
// if it's a scala object, we can simply keep the full class path.
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index 0978e92..277584b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -19,13 +19,16 @@ package org.apache.spark.sql.catalyst
import java.io._
import java.nio.charset.StandardCharsets
+import java.util.concurrent.atomic.AtomicBoolean
+import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{NumericType, StringType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
-package object util {
+package object util extends Logging {
/** Silences output to stderr or stdout for the duration of f */
def quietly[A](f: => A): A = {
@@ -167,6 +170,38 @@ package object util {
builder.toString()
}
+ /** Whether we have warned about plan string truncation yet. */
+ private val truncationWarningPrinted = new AtomicBoolean(false)
+
+ /**
+ * Format a sequence with semantics similar to calling .mkString(). Any elements beyond
+ * maxNumToStringFields will be dropped and replaced by a "... N more fields" placeholder.
+ *
+ * @return the trimmed and formatted string.
+ */
+ def truncatedString[T](
+ seq: Seq[T],
+ start: String,
+ sep: String,
+ end: String,
+ maxNumFields: Int = SQLConf.get.maxToStringFields): String = {
+ if (seq.length > maxNumFields) {
+ if (truncationWarningPrinted.compareAndSet(false, true)) {
+ logWarning(
+ "Truncated the string representation of a plan since it was too large. This " +
+ s"behavior can be adjusted by setting '${SQLConf.MAX_TO_STRING_FIELDS.key}'.")
+ }
+ val numFields = math.max(0, maxNumFields - 1)
+ seq.take(numFields).mkString(
+ start, sep, sep + "... " + (seq.length - numFields) + " more fields" + end)
+ } else {
+ seq.mkString(start, sep, end)
+ }
+ }
+
+ /** Shorthand for calling truncatedString() without start or end strings. */
+ def truncatedString[T](seq: Seq[T], sep: String): String = truncatedString(seq, "", sep, "")
+
/* FIX ME
implicit class debugLogging(a: Any) {
def debugLogging() {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 518115d..cc0e972 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1594,6 +1594,13 @@ object SQLConf {
"WHERE, which does not follow SQL standard.")
.booleanConf
.createWithDefault(false)
+
+ val MAX_TO_STRING_FIELDS = buildConf("spark.sql.debug.maxToStringFields")
+ .doc("Maximum number of fields of sequence-like entries can be converted to strings " +
+ "in debug output. Any elements beyond the limit will be dropped and replaced by a" +
+ """ "... N more fields" placeholder.""")
+ .intConf
+ .createWithDefault(25)
}
/**
@@ -2009,6 +2016,8 @@ class SQLConf extends Serializable with Logging {
def integralDivideReturnLong: Boolean = getConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG)
+ def maxToStringFields: Int = getConf(SQLConf.MAX_TO_STRING_FIELDS)
+
/** ********************** SQLConf functionality methods ************ */
/** Set Spark SQL configuration properties. */
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index 3bef75d..6e8bbde 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkException
import org.apache.spark.annotation.Stable
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, InterpretedOrdering}
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, LegacyTypeStringParser}
-import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.catalyst.util.{quoteIdentifier, truncatedString}
import org.apache.spark.util.Utils
/**
@@ -346,7 +346,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
override def simpleString: String = {
val fieldTypes = fields.view.map(field => s"${field.name}:${field.dataType.simpleString}")
- Utils.truncatedString(fieldTypes, "struct<", ",", ">")
+ truncatedString(fieldTypes, "struct<", ",", ">")
}
override def catalogString: String = {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/catalyst/src/test/scala/org/apache/spark/sql/util/UtilSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/UtilSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/UtilSuite.scala
new file mode 100644
index 0000000..9c16202
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/UtilSuite.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.util.truncatedString
+
+class UtilSuite extends SparkFunSuite {
+ test("truncatedString") {
+ assert(truncatedString(Nil, "[", ", ", "]", 2) == "[]")
+ assert(truncatedString(Seq(1, 2), "[", ", ", "]", 2) == "[1, 2]")
+ assert(truncatedString(Seq(1, 2, 3), "[", ", ", "]", 2) == "[1, ... 2 more fields]")
+ assert(truncatedString(Seq(1, 2, 3), "[", ", ", "]", -5) == "[, ... 3 more fields]")
+ assert(truncatedString(Seq(1, 2, 3), ", ") == "1, 2, 3")
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index a9b18ab..77e381e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, UnknownPartitioning}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.datasources._
import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat => ParquetSource}
import org.apache.spark.sql.execution.metric.SQLMetrics
@@ -56,8 +57,8 @@ trait DataSourceScanExec extends LeafExecNode with CodegenSupport {
case (key, value) =>
key + ": " + StringUtils.abbreviate(redact(value), 100)
}
- val metadataStr = Utils.truncatedString(metadataEntries, " ", ", ", "")
- s"$nodeNamePrefix$nodeName${Utils.truncatedString(output, "[", ",", "]")}$metadataStr"
+ val metadataStr = truncatedString(metadataEntries, " ", ", ", "")
+ s"$nodeNamePrefix$nodeName${truncatedString(output, "[", ",", "]")}$metadataStr"
}
override def verboseString: String = redact(super.verboseString)
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/ExistingRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExistingRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExistingRDD.scala
index 2962bec..9f67d55 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExistingRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExistingRDD.scala
@@ -24,9 +24,9 @@ import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.metric.SQLMetrics
import org.apache.spark.sql.types.DataType
-import org.apache.spark.util.Utils
object RDDConversions {
def productToRowRdd[A <: Product](data: RDD[A], outputTypes: Seq[DataType]): RDD[InternalRow] = {
@@ -197,6 +197,6 @@ case class RDDScanExec(
}
override def simpleString: String = {
- s"$nodeName${Utils.truncatedString(output, "[", ",", "]")}"
+ s"$nodeName${truncatedString(output, "[", ",", "]")}"
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 87a4ceb..cfb5e43 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.analysis.UnsupportedOperationChecker
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ReturnAnswer}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.command.{DescribeTableCommand, ExecutedCommandExec, ShowTablesCommand}
import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReuseExchange}
import org.apache.spark.sql.types.{BinaryType, DateType, DecimalType, TimestampType, _}
@@ -213,7 +214,7 @@ class QueryExecution(
writer.write("== Parsed Logical Plan ==\n")
writeOrError(writer)(logical.treeString(_, verbose, addSuffix))
writer.write("\n== Analyzed Logical Plan ==\n")
- val analyzedOutput = stringOrError(Utils.truncatedString(
+ val analyzedOutput = stringOrError(truncatedString(
analyzed.output.map(o => s"${o.name}: ${o.dataType.simpleString}"), ", "))
writer.write(analyzedOutput)
writer.write("\n")
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index 08dcdf3..4827f83 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.plans.physical._
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
import org.apache.spark.sql.execution.vectorized.MutableColumnarRow
@@ -930,9 +931,9 @@ case class HashAggregateExec(
testFallbackStartsAt match {
case None =>
- val keyString = Utils.truncatedString(groupingExpressions, "[", ", ", "]")
- val functionString = Utils.truncatedString(allAggregateExpressions, "[", ", ", "]")
- val outputString = Utils.truncatedString(output, "[", ", ", "]")
+ val keyString = truncatedString(groupingExpressions, "[", ", ", "]")
+ val functionString = truncatedString(allAggregateExpressions, "[", ", ", "]")
+ val outputString = truncatedString(output, "[", ", ", "]")
if (verbose) {
s"HashAggregate(keys=$keyString, functions=$functionString, output=$outputString)"
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
index 66955b8..7145bb0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
@@ -23,9 +23,9 @@ import org.apache.spark.sql.catalyst.errors._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.physical._
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.util.Utils
/**
* A hash-based aggregate operator that supports [[TypedImperativeAggregate]] functions that may
@@ -143,9 +143,9 @@ case class ObjectHashAggregateExec(
private def toString(verbose: Boolean): String = {
val allAggregateExpressions = aggregateExpressions
- val keyString = Utils.truncatedString(groupingExpressions, "[", ", ", "]")
- val functionString = Utils.truncatedString(allAggregateExpressions, "[", ", ", "]")
- val outputString = Utils.truncatedString(output, "[", ", ", "]")
+ val keyString = truncatedString(groupingExpressions, "[", ", ", "]")
+ val functionString = truncatedString(allAggregateExpressions, "[", ", ", "]")
+ val outputString = truncatedString(output, "[", ", ", "]")
if (verbose) {
s"ObjectHashAggregate(keys=$keyString, functions=$functionString, output=$outputString)"
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
index fc87de2..d732b90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
@@ -23,9 +23,9 @@ import org.apache.spark.sql.catalyst.errors._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.physical._
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.util.Utils
/**
* Sort-based aggregate operator.
@@ -114,9 +114,9 @@ case class SortAggregateExec(
private def toString(verbose: Boolean): String = {
val allAggregateExpressions = aggregateExpressions
- val keyString = Utils.truncatedString(groupingExpressions, "[", ", ", "]")
- val functionString = Utils.truncatedString(allAggregateExpressions, "[", ", ", "]")
- val outputString = Utils.truncatedString(output, "[", ", ", "]")
+ val keyString = truncatedString(groupingExpressions, "[", ", ", "]")
+ val functionString = truncatedString(allAggregateExpressions, "[", ", ", "]")
+ val outputString = truncatedString(output, "[", ", ", "]")
if (verbose) {
s"SortAggregate(key=$keyString, functions=$functionString, output=$outputString)"
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
index 3b65885..73eb65f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
@@ -27,9 +27,10 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.plans.logical.{HintInfo, LogicalPlan, Statistics}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.storage.StorageLevel
-import org.apache.spark.util.{LongAccumulator, Utils}
+import org.apache.spark.util.LongAccumulator
/**
@@ -209,5 +210,5 @@ case class InMemoryRelation(
override protected def otherCopyArgs: Seq[AnyRef] = Seq(statsOfPlanToCache)
override def simpleString: String =
- s"InMemoryRelation [${Utils.truncatedString(output, ", ")}], ${cacheBuilder.storageLevel}"
+ s"InMemoryRelation [${truncatedString(output, ", ")}], ${cacheBuilder.storageLevel}"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
index 8d715f6..1023572 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
@@ -21,8 +21,8 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.expressions.{AttributeMap, AttributeReference}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.sources.BaseRelation
-import org.apache.spark.util.Utils
/**
* Used to link a [[BaseRelation]] in to a logical query plan.
@@ -63,7 +63,7 @@ case class LogicalRelation(
case _ => // Do nothing.
}
- override def simpleString: String = s"Relation[${Utils.truncatedString(output, ",")}] $relation"
+ override def simpleString: String = s"Relation[${truncatedString(output, ",")}] $relation"
}
object LogicalRelation {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index f150144..51c385e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -27,10 +27,10 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode, SparkSession, SQLContext}
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.jdbc.JdbcDialects
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{DataType, DateType, NumericType, StructType, TimestampType}
-import org.apache.spark.util.Utils
/**
* Instructions on how to partition the table among workers.
@@ -159,8 +159,9 @@ private[sql] object JDBCRelation extends Logging {
val column = schema.find { f =>
resolver(f.name, columnName) || resolver(dialect.quoteIdentifier(f.name), columnName)
}.getOrElse {
+ val maxNumToStringFields = SQLConf.get.maxToStringFields
throw new AnalysisException(s"User-defined partition column $columnName not " +
- s"found in the JDBC relation: ${schema.simpleString(Utils.maxNumToStringFields)}")
+ s"found in the JDBC relation: ${schema.simpleString(maxNumToStringFields)}")
}
column.dataType match {
case _: NumericType | DateType | TimestampType =>
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StringFormat.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StringFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StringFormat.scala
index 97e6c6d..e829f62 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StringFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StringFormat.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.v2
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.sources.DataSourceRegister
import org.apache.spark.sql.sources.v2.DataSourceV2
import org.apache.spark.util.Utils
@@ -72,10 +73,10 @@ trait DataSourceV2StringFormat {
}.mkString("[", ",", "]")
}
- val outputStr = Utils.truncatedString(output, "[", ", ", "]")
+ val outputStr = truncatedString(output, "[", ", ", "]")
val entriesStr = if (entries.nonEmpty) {
- Utils.truncatedString(entries.map {
+ truncatedString(entries.map {
case (key, value) => key + ": " + StringUtils.abbreviate(value, 100)
}, " (", ", ", ")")
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
index 9bfe1a7..90dafcf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
@@ -23,8 +23,8 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, LazilyGeneratedOrdering}
import org.apache.spark.sql.catalyst.plans.physical._
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-import org.apache.spark.util.Utils
/**
* Take the first `limit` elements and collect them to a single partition.
@@ -177,8 +177,8 @@ case class TakeOrderedAndProjectExec(
override def outputPartitioning: Partitioning = SinglePartition
override def simpleString: String = {
- val orderByString = Utils.truncatedString(sortOrder, "[", ",", "]")
- val outputString = Utils.truncatedString(output, "[", ",", "]")
+ val orderByString = truncatedString(sortOrder, "[", ",", "]")
+ val outputString = truncatedString(output, "[", ",", "]")
s"TakeOrderedAndProject(limit=$limit, orderBy=$orderByString, output=$outputString)"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index 2cac865..5defca3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -24,13 +24,14 @@ import org.apache.spark.sql.{Dataset, SparkSession}
import org.apache.spark.sql.catalyst.encoders.RowEncoder
import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentBatchTimestamp, CurrentDate, CurrentTimestamp}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.SQLExecution
import org.apache.spark.sql.execution.datasources.v2.{StreamingDataSourceV2Relation, WriteToDataSourceV2}
import org.apache.spark.sql.execution.streaming.sources.{MicroBatchWritSupport, RateControlMicroBatchReadSupport}
import org.apache.spark.sql.sources.v2._
import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReadSupport, Offset => OffsetV2}
import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime, Trigger}
-import org.apache.spark.util.{Clock, Utils}
+import org.apache.spark.util.Clock
class MicroBatchExecution(
sparkSession: SparkSession,
@@ -475,8 +476,8 @@ class MicroBatchExecution(
case StreamingExecutionRelation(source, output) =>
newData.get(source).map { dataPlan =>
assert(output.size == dataPlan.output.size,
- s"Invalid batch: ${Utils.truncatedString(output, ",")} != " +
- s"${Utils.truncatedString(dataPlan.output, ",")}")
+ s"Invalid batch: ${truncatedString(output, ",")} != " +
+ s"${truncatedString(dataPlan.output, ",")}")
val aliases = output.zip(dataPlan.output).map { case (to, from) =>
Alias(from, to.name)(exprId = to.exprId, explicitMetadata = Some(from.metadata))
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
index 4a7df73..1eab551 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
@@ -28,6 +28,7 @@ import org.apache.spark.SparkEnv
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, CurrentDate, CurrentTimestamp}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.execution.SQLExecution
import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanExec, StreamingDataSourceV2Relation}
import org.apache.spark.sql.execution.streaming.{ContinuousExecutionRelation, StreamingRelationV2, _}
@@ -35,7 +36,7 @@ import org.apache.spark.sql.sources.v2
import org.apache.spark.sql.sources.v2.{ContinuousReadSupportProvider, DataSourceOptions, StreamingWriteSupportProvider}
import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReadSupport, PartitionOffset}
import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime, Trigger}
-import org.apache.spark.util.{Clock, Utils}
+import org.apache.spark.util.Clock
class ContinuousExecution(
sparkSession: SparkSession,
@@ -164,8 +165,8 @@ class ContinuousExecution(
val newOutput = readSupport.fullSchema().toAttributes
assert(output.size == newOutput.size,
- s"Invalid reader: ${Utils.truncatedString(output, ",")} != " +
- s"${Utils.truncatedString(newOutput, ",")}")
+ s"Invalid reader: ${truncatedString(output, ",")} != " +
+ s"${truncatedString(newOutput, ",")}")
replacements ++= output.zip(newOutput)
val loggedOffset = offsets.offsets(0)
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index adf52ab..daee089 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -31,11 +31,11 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeRow}
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
import org.apache.spark.sql.catalyst.streaming.InternalOutputModes._
+import org.apache.spark.sql.catalyst.util.truncatedString
import org.apache.spark.sql.sources.v2.reader._
import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReadSupport, Offset => OffsetV2}
import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
-import org.apache.spark.util.Utils
object MemoryStream {
protected val currentBlockId = new AtomicInteger(0)
@@ -117,7 +117,7 @@ case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
}
}
- override def toString: String = s"MemoryStream[${Utils.truncatedString(output, ",")}]"
+ override def toString: String = s"MemoryStream[${truncatedString(output, ",")}]"
override def deserializeOffset(json: String): OffsetV2 = LongOffset(json.toLong)
http://git-wip-us.apache.org/repos/asf/spark/blob/81550b38/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
index a5922d7..0c47a20 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala
@@ -20,9 +20,20 @@ import scala.io.Source
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
+case class QueryExecutionTestRecord(
+ c0: Int, c1: Int, c2: Int, c3: Int, c4: Int,
+ c5: Int, c6: Int, c7: Int, c8: Int, c9: Int,
+ c10: Int, c11: Int, c12: Int, c13: Int, c14: Int,
+ c15: Int, c16: Int, c17: Int, c18: Int, c19: Int,
+ c20: Int, c21: Int, c22: Int, c23: Int, c24: Int,
+ c25: Int, c26: Int)
+
class QueryExecutionSuite extends SharedSQLContext {
+ import testImplicits._
+
def checkDumpedPlans(path: String, expected: Int): Unit = {
assert(Source.fromFile(path).getLines.toList
.takeWhile(_ != "== Whole Stage Codegen ==") == List(
@@ -80,6 +91,21 @@ class QueryExecutionSuite extends SharedSQLContext {
assert(exception.getMessage.contains("Illegal character in scheme name"))
}
+ test("limit number of fields by sql config") {
+ def relationPlans: String = {
+ val ds = spark.createDataset(Seq(QueryExecutionTestRecord(
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)))
+ ds.queryExecution.toString
+ }
+ withSQLConf(SQLConf.MAX_TO_STRING_FIELDS.key -> "26") {
+ assert(relationPlans.contains("more fields"))
+ }
+ withSQLConf(SQLConf.MAX_TO_STRING_FIELDS.key -> "27") {
+ assert(!relationPlans.contains("more fields"))
+ }
+ }
+
test("toString() exception/error handling") {
spark.experimental.extraStrategies = Seq(
new SparkStrategy {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org