You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/07/20 06:29:34 UTC
[1/2] spark git commit: [SPARK-24268][SQL] Use datatype.catalogString
in error messages
Repository: spark
Updated Branches:
refs/heads/master 1462b1766 -> a5925c163
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index d5969b5..31ef090 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -244,7 +244,7 @@ public class VectorizedColumnReader {
return new SchemaColumnConvertNotSupportedException(
Arrays.toString(descriptor.getPath()),
descriptor.getType().toString(),
- column.dataType().toString());
+ column.dataType().catalogString());
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index c6449cd..b068493 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -452,7 +452,7 @@ class RelationalGroupedDataset protected[sql](
require(expr.evalType == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF,
"Must pass a grouped map udf")
require(expr.dataType.isInstanceOf[StructType],
- "The returnType of the udf must be a StructType")
+ s"The returnType of the udf must be a ${StructType.simpleString}")
val groupingNamedExpressions = groupingExprs.map {
case ne: NamedExpression => ne
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
index 93c8127..533097a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
@@ -47,11 +47,13 @@ object ArrowUtils {
case DateType => new ArrowType.Date(DateUnit.DAY)
case TimestampType =>
if (timeZoneId == null) {
- throw new UnsupportedOperationException("TimestampType must supply timeZoneId parameter")
+ throw new UnsupportedOperationException(
+ s"${TimestampType.catalogString} must supply timeZoneId parameter")
} else {
new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
}
- case _ => throw new UnsupportedOperationException(s"Unsupported data type: ${dt.simpleString}")
+ case _ =>
+ throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
}
def fromArrowType(dt: ArrowType): DataType = dt match {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
index 66888fc..3de6ea8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
@@ -68,7 +68,7 @@ object ArrowWriter {
}
new StructWriter(vector, children.toArray)
case (dt, _) =>
- throw new UnsupportedOperationException(s"Unsupported data type: ${dt.simpleString}")
+ throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
index e9b150f..542a10f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala
@@ -717,7 +717,7 @@ private[columnar] object ColumnType {
case struct: StructType => STRUCT(struct)
case udt: UserDefinedType[_] => apply(udt.sqlType)
case other =>
- throw new Exception(s"Unsupported type: ${other.simpleString}")
+ throw new Exception(s"Unsupported type: ${other.catalogString}")
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index 82e9919..cccd6c0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -45,7 +45,7 @@ object DataSourceUtils {
schema.foreach { field =>
if (!format.supportDataType(field.dataType, isReadPath)) {
throw new AnalysisException(
- s"$format data source does not support ${field.dataType.simpleString} data type.")
+ s"$format data source does not support ${field.dataType.catalogString} data type.")
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index b81737e..6cc7922 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -175,7 +175,7 @@ object JdbcUtils extends Logging {
private def getJdbcType(dt: DataType, dialect: JdbcDialect): JdbcType = {
dialect.getJDBCType(dt).orElse(getCommonJDBCType(dt)).getOrElse(
- throw new IllegalArgumentException(s"Can't get JDBC type for ${dt.simpleString}"))
+ throw new IllegalArgumentException(s"Can't get JDBC type for ${dt.catalogString}"))
}
/**
@@ -480,7 +480,7 @@ object JdbcUtils extends Logging {
case LongType if metadata.contains("binarylong") =>
throw new IllegalArgumentException(s"Unsupported array element " +
- s"type ${dt.simpleString} based on binary")
+ s"type ${dt.catalogString} based on binary")
case ArrayType(_, _) =>
throw new IllegalArgumentException("Nested arrays unsupported")
@@ -494,7 +494,7 @@ object JdbcUtils extends Logging {
array => new GenericArrayData(elementConversion.apply(array.getArray)))
row.update(pos, array)
- case _ => throw new IllegalArgumentException(s"Unsupported type ${dt.simpleString}")
+ case _ => throw new IllegalArgumentException(s"Unsupported type ${dt.catalogString}")
}
private def nullSafeConvert[T](input: T, f: T => Any): Any = {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 4f44ae4..c4c3b30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -98,7 +98,7 @@ private[orc] object OrcFilters {
case DateType => PredicateLeaf.Type.DATE
case TimestampType => PredicateLeaf.Type.TIMESTAMP
case _: DecimalType => PredicateLeaf.Type.DECIMAL
- case _ => throw new UnsupportedOperationException(s"DataType: $dataType")
+ case _ => throw new UnsupportedOperationException(s"DataType: ${dataType.catalogString}")
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index 460194b..b404cfa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -104,7 +104,7 @@ object OrcUtils extends Logging {
// This is a ORC file written by Hive, no field names in the physical schema, assume the
// physical schema maps to the data scheme by index.
assert(orcFieldNames.length <= dataSchema.length, "The given data schema " +
- s"${dataSchema.simpleString} has less fields than the actual ORC physical schema, " +
+ s"${dataSchema.catalogString} has less fields than the actual ORC physical schema, " +
"no idea which columns were dropped, fail to read.")
Some(requiredSchema.fieldNames.map { name =>
val index = dataSchema.fieldIndex(name)
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
index c61be07..70f42f2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
@@ -555,7 +555,7 @@ class SparkToParquetSchemaConverter(
convertField(field.copy(dataType = udt.sqlType))
case _ =>
- throw new AnalysisException(s"Unsupported data type $field.dataType")
+ throw new AnalysisException(s"Unsupported data type ${field.dataType.catalogString}")
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index cab0025..dfcf6c1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -281,7 +281,7 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
schema.filter(f => normalizedPartitionCols.contains(f.name)).map(_.dataType).foreach {
case _: AtomicType => // OK
- case other => failAnalysis(s"Cannot use ${other.simpleString} for partition column")
+ case other => failAnalysis(s"Cannot use ${other.catalogString} for partition column")
}
normalizedPartitionCols
@@ -307,7 +307,7 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
normalizedBucketSpec.sortColumnNames.map(schema(_)).map(_.dataType).foreach {
case dt if RowOrdering.isOrderable(dt) => // OK
- case other => failAnalysis(s"Cannot use ${other.simpleString} for sorting column")
+ case other => failAnalysis(s"Cannot use ${other.catalogString} for sorting column")
}
Some(normalizedBucketSpec)
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
index 685d584..bea652c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
@@ -157,7 +157,7 @@ object StatFunctions extends Logging {
cols.map(name => (name, df.schema.fields.find(_.name == name))).foreach { case (name, data) =>
require(data.nonEmpty, s"Couldn't find column with name $name")
require(data.get.dataType.isInstanceOf[NumericType], s"Currently $functionName calculation " +
- s"for columns with dataType ${data.get.dataType} not supported.")
+ s"for columns with dataType ${data.get.dataType.catalogString} not supported.")
}
val columns = cols.map(n => Column(Cast(Column(n).expr, DoubleType)))
df.select(columns: _*).queryExecution.toRdd.treeAggregate(new CovarianceCounter)(
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
index 3d49323..827931d 100644
--- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
@@ -120,7 +120,7 @@ select to_json(named_struct('a', 1, 'b', 2), map('mode', 1))
struct<>
-- !query 11 output
org.apache.spark.sql.AnalysisException
-A type of keys and values in map() must be string, but got MapType(StringType,IntegerType,false);; line 1 pos 7
+A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
-- !query 12
@@ -216,7 +216,7 @@ select from_json('{"a":1}', 'a INT', map('mode', 1))
struct<>
-- !query 20 output
org.apache.spark.sql.AnalysisException
-A type of keys and values in map() must be string, but got MapType(StringType,IntegerType,false);; line 1 pos 7
+A type of keys and values in map() must be string, but got map<string,int>;; line 1 pos 7
-- !query 21
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/test/resources/sql-tests/results/literals.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
index b8c91dc..7f30161 100644
--- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
@@ -147,7 +147,7 @@ struct<>
-- !query 15 output
org.apache.spark.sql.catalyst.parser.ParseException
-DecimalType can only support precision up to 38
+decimal can only support precision up to 38
== SQL ==
select 1234567890123456789012345678901234567890
@@ -159,7 +159,7 @@ struct<>
-- !query 16 output
org.apache.spark.sql.catalyst.parser.ParseException
-DecimalType can only support precision up to 38
+decimal can only support precision up to 38
== SQL ==
select 1234567890123456789012345678901234567890.0
@@ -379,7 +379,7 @@ struct<>
-- !query 39 output
org.apache.spark.sql.catalyst.parser.ParseException
-DecimalType can only support precision up to 38(line 1, pos 7)
+decimal can only support precision up to 38(line 1, pos 7)
== SQL ==
select 1.20E-38BD
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index a7ce952..9f9af89 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -312,14 +312,14 @@ class FileBasedDataSourceSuite extends QueryTest with SharedSQLContext with Befo
Seq((1, new UDT.MyDenseVector(Array(0.25, 2.25, 4.25)))).toDF("id", "vectors")
.write.mode("overwrite").csv(csvDir)
}.getMessage
- assert(msg.contains("CSV data source does not support mydensevector data type"))
+ assert(msg.contains("CSV data source does not support array<double> data type"))
msg = intercept[AnalysisException] {
val schema = StructType(StructField("a", new UDT.MyDenseVectorUDT(), true) :: Nil)
spark.range(1).write.mode("overwrite").csv(csvDir)
spark.read.schema(schema).csv(csvDir).collect()
}.getMessage
- assert(msg.contains("CSV data source does not support mydensevector data type."))
+ assert(msg.contains("CSV data source does not support array<double> data type."))
}
}
@@ -339,7 +339,7 @@ class FileBasedDataSourceSuite extends QueryTest with SharedSQLContext with Befo
sql("select testType()").write.format(format).mode("overwrite").save(tempDir)
}.getMessage
assert(msg.toLowerCase(Locale.ROOT)
- .contains(s"$format data source does not support interval data type."))
+ .contains(s"$format data source does not support calendarinterval data type."))
}
// read path
@@ -358,7 +358,7 @@ class FileBasedDataSourceSuite extends QueryTest with SharedSQLContext with Befo
spark.read.schema(schema).format(format).load(tempDir).collect()
}.getMessage
assert(msg.toLowerCase(Locale.ROOT)
- .contains(s"$format data source does not support interval data type."))
+ .contains(s"$format data source does not support calendarinterval data type."))
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
index 9d3dfae..368e52c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -430,9 +430,9 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
val col = spark.read.parquet(file).schema.fields.filter(_.name.equals("a"))
assert(col.length == 1)
if (col(0).dataType == StringType) {
- assert(errMsg.contains("Column: [a], Expected: IntegerType, Found: BINARY"))
+ assert(errMsg.contains("Column: [a], Expected: int, Found: BINARY"))
} else {
- assert(errMsg.endsWith("Column: [a], Expected: StringType, Found: INT32"))
+ assert(errMsg.endsWith("Column: [a], Expected: string, Found: INT32"))
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 7f28fc4..5cc1047 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -785,9 +785,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
// schema we read back is different(ignore case and nullability) from the one in table
// properties which was written when creating table, we should respect the table schema
// from hive.
- logWarning(s"The table schema given by Hive metastore(${table.schema.simpleString}) is " +
+ logWarning(s"The table schema given by Hive metastore(${table.schema.catalogString}) is " +
"different from the schema when this table was created by Spark SQL" +
- s"(${schemaFromTableProps.simpleString}). We have to fall back to the table schema " +
+ s"(${schemaFromTableProps.catalogString}). We have to fall back to the table schema " +
"from Hive metastore which is not case preserving.")
hiveTable.copy(schemaPreservesCase = false)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
index 7dcaf17..6052486 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
@@ -78,9 +78,9 @@ case class HiveTableScanExec(
// Bind all partition key attribute references in the partition pruning predicate for later
// evaluation.
private lazy val boundPruningPred = partitionPruningPred.reduceLeftOption(And).map { pred =>
- require(
- pred.dataType == BooleanType,
- s"Data type of predicate $pred must be BooleanType rather than ${pred.dataType}.")
+ require(pred.dataType == BooleanType,
+ s"Data type of predicate $pred must be ${BooleanType.catalogString} rather than " +
+ s"${pred.dataType.catalogString}.")
BindReferences.bindReference(pred, relation.partitionCols)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index fb4957e..d84f9a3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -155,7 +155,7 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
spark.udf.register("testType", () => new IntervalData())
sql("select testType()").write.mode("overwrite").orc(orcDir)
}.getMessage
- assert(msg.contains("ORC data source does not support interval data type."))
+ assert(msg.contains("ORC data source does not support calendarinterval data type."))
// read path
msg = intercept[AnalysisException] {
@@ -170,7 +170,7 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
spark.range(1).write.mode("overwrite").orc(orcDir)
spark.read.schema(schema).orc(orcDir).collect()
}.getMessage
- assert(msg.contains("ORC data source does not support interval data type."))
+ assert(msg.contains("ORC data source does not support calendarinterval data type."))
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/2] spark git commit: [SPARK-24268][SQL] Use datatype.catalogString
in error messages
Posted by li...@apache.org.
[SPARK-24268][SQL] Use datatype.catalogString in error messages
## What changes were proposed in this pull request?
As stated in https://github.com/apache/spark/pull/21321, in the error messages we should use `catalogString`. This is not the case, as SPARK-22893 used `simpleString` in order to have the same representation everywhere and it missed some places.
The PR unifies the messages using alway the `catalogString` representation of the dataTypes in the messages.
## How was this patch tested?
existing/modified UTs
Author: Marco Gaido <ma...@gmail.com>
Closes #21804 from mgaido91/SPARK-24268_catalog.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a5925c16
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a5925c16
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a5925c16
Branch: refs/heads/master
Commit: a5925c1631e25c2dcc3c2948cea31e993ce66a97
Parents: 1462b17
Author: Marco Gaido <ma...@gmail.com>
Authored: Thu Jul 19 23:29:29 2018 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Thu Jul 19 23:29:29 2018 -0700
----------------------------------------------------------------------
.../spark/sql/kafka010/KafkaWriteTask.scala | 6 +++---
.../apache/spark/sql/kafka010/KafkaWriter.scala | 6 +++---
.../sql/kafka010/KafkaContinuousSinkSuite.scala | 4 ++--
.../spark/sql/kafka010/KafkaSinkSuite.scala | 4 ++--
.../scala/org/apache/spark/ml/feature/DCT.scala | 3 ++-
.../apache/spark/ml/feature/FeatureHasher.scala | 5 +++--
.../org/apache/spark/ml/feature/HashingTF.scala | 2 +-
.../org/apache/spark/ml/feature/Interaction.scala | 3 ++-
.../scala/org/apache/spark/ml/feature/NGram.scala | 3 ++-
.../apache/spark/ml/feature/OneHotEncoder.scala | 3 ++-
.../org/apache/spark/ml/feature/RFormula.scala | 2 +-
.../spark/ml/feature/StopWordsRemover.scala | 4 ++--
.../org/apache/spark/ml/feature/Tokenizer.scala | 3 ++-
.../apache/spark/ml/feature/VectorAssembler.scala | 2 +-
.../scala/org/apache/spark/ml/fpm/FPGrowth.scala | 2 +-
.../org/apache/spark/ml/util/SchemaUtils.scala | 11 +++++++----
.../BinaryClassificationEvaluatorSuite.scala | 4 ++--
.../apache/spark/ml/feature/RFormulaSuite.scala | 2 +-
.../spark/ml/feature/VectorAssemblerSuite.scala | 6 +++---
.../apache/spark/ml/recommendation/ALSSuite.scala | 2 +-
.../regression/AFTSurvivalRegressionSuite.scala | 2 +-
.../org/apache/spark/ml/util/MLTestingUtils.scala | 6 +++---
.../spark/sql/catalyst/analysis/Analyzer.scala | 8 ++++----
.../sql/catalyst/analysis/CheckAnalysis.scala | 18 +++++++++---------
.../apache/spark/sql/catalyst/analysis/view.scala | 3 ++-
.../spark/sql/catalyst/expressions/Cast.scala | 2 +-
.../catalyst/expressions/ExpectsInputTypes.scala | 2 +-
.../sql/catalyst/expressions/Expression.scala | 4 ++--
.../spark/sql/catalyst/expressions/ScalaUDF.scala | 5 +++--
.../sql/catalyst/expressions/SortOrder.scala | 2 +-
.../aggregate/ApproximatePercentile.scala | 4 ++--
.../sql/catalyst/expressions/arithmetic.scala | 4 ++--
.../expressions/codegen/CodeGenerator.scala | 4 ++--
.../expressions/collectionOperations.scala | 14 +++++++-------
.../catalyst/expressions/complexTypeCreator.scala | 8 ++++----
.../expressions/complexTypeExtractors.scala | 2 +-
.../expressions/conditionalExpressions.scala | 4 ++--
.../sql/catalyst/expressions/generators.scala | 8 ++++----
.../catalyst/expressions/jsonExpressions.scala | 6 +++---
.../catalyst/expressions/objects/objects.scala | 2 +-
.../sql/catalyst/expressions/predicates.scala | 2 +-
.../catalyst/expressions/stringExpressions.scala | 7 ++++---
.../catalyst/expressions/windowExpressions.scala | 6 +++---
.../sql/catalyst/json/JacksonGenerator.scala | 8 ++++----
.../spark/sql/catalyst/json/JacksonParser.scala | 6 ++++--
.../spark/sql/catalyst/json/JacksonUtils.scala | 2 +-
.../spark/sql/catalyst/json/JsonInferSchema.scala | 6 ++++--
.../spark/sql/catalyst/util/TypeUtils.scala | 7 ++++---
.../apache/spark/sql/types/AbstractDataType.scala | 9 +++++----
.../org/apache/spark/sql/types/ArrayType.scala | 5 +++--
.../org/apache/spark/sql/types/DecimalType.scala | 3 ++-
.../org/apache/spark/sql/types/ObjectType.scala | 3 ++-
.../org/apache/spark/sql/types/StructType.scala | 5 +++--
.../catalyst/analysis/AnalysisErrorSuite.scala | 2 +-
.../analysis/ExpressionTypeCheckingSuite.scala | 16 ++++++++--------
.../catalyst/parser/ExpressionParserSuite.scala | 2 +-
.../apache/spark/sql/types/DataTypeSuite.scala | 2 +-
.../parquet/VectorizedColumnReader.java | 2 +-
.../spark/sql/RelationalGroupedDataset.scala | 2 +-
.../spark/sql/execution/arrow/ArrowUtils.scala | 6 ++++--
.../spark/sql/execution/arrow/ArrowWriter.scala | 2 +-
.../spark/sql/execution/columnar/ColumnType.scala | 2 +-
.../execution/datasources/DataSourceUtils.scala | 2 +-
.../execution/datasources/jdbc/JdbcUtils.scala | 6 +++---
.../execution/datasources/orc/OrcFilters.scala | 2 +-
.../sql/execution/datasources/orc/OrcUtils.scala | 2 +-
.../parquet/ParquetSchemaConverter.scala | 2 +-
.../spark/sql/execution/datasources/rules.scala | 4 ++--
.../spark/sql/execution/stat/StatFunctions.scala | 2 +-
.../sql-tests/results/json-functions.sql.out | 4 ++--
.../resources/sql-tests/results/literals.sql.out | 6 +++---
.../spark/sql/FileBasedDataSourceSuite.scala | 8 ++++----
.../datasources/parquet/ParquetSchemaSuite.scala | 4 ++--
.../spark/sql/hive/HiveExternalCatalog.scala | 4 ++--
.../sql/hive/execution/HiveTableScanExec.scala | 6 +++---
.../spark/sql/hive/orc/HiveOrcSourceSuite.scala | 4 ++--
76 files changed, 185 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala
index d90630a..041fac7 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriteTask.scala
@@ -110,7 +110,7 @@ private[kafka010] abstract class KafkaRowWriter(
case t =>
throw new IllegalStateException(s"${KafkaWriter.TOPIC_ATTRIBUTE_NAME} " +
s"attribute unsupported type $t. ${KafkaWriter.TOPIC_ATTRIBUTE_NAME} " +
- "must be a StringType")
+ s"must be a ${StringType.catalogString}")
}
val keyExpression = inputSchema.find(_.name == KafkaWriter.KEY_ATTRIBUTE_NAME)
.getOrElse(Literal(null, BinaryType))
@@ -118,7 +118,7 @@ private[kafka010] abstract class KafkaRowWriter(
case StringType | BinaryType => // good
case t =>
throw new IllegalStateException(s"${KafkaWriter.KEY_ATTRIBUTE_NAME} " +
- s"attribute unsupported type $t")
+ s"attribute unsupported type ${t.catalogString}")
}
val valueExpression = inputSchema
.find(_.name == KafkaWriter.VALUE_ATTRIBUTE_NAME).getOrElse(
@@ -129,7 +129,7 @@ private[kafka010] abstract class KafkaRowWriter(
case StringType | BinaryType => // good
case t =>
throw new IllegalStateException(s"${KafkaWriter.VALUE_ATTRIBUTE_NAME} " +
- s"attribute unsupported type $t")
+ s"attribute unsupported type ${t.catalogString}")
}
UnsafeProjection.create(
Seq(topicExpression, Cast(keyExpression, BinaryType),
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriter.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriter.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriter.scala
index 15cd448..fc09938 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriter.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaWriter.scala
@@ -57,7 +57,7 @@ private[kafka010] object KafkaWriter extends Logging {
).dataType match {
case StringType => // good
case _ =>
- throw new AnalysisException(s"Topic type must be a String")
+ throw new AnalysisException(s"Topic type must be a ${StringType.catalogString}")
}
schema.find(_.name == KEY_ATTRIBUTE_NAME).getOrElse(
Literal(null, StringType)
@@ -65,7 +65,7 @@ private[kafka010] object KafkaWriter extends Logging {
case StringType | BinaryType => // good
case _ =>
throw new AnalysisException(s"$KEY_ATTRIBUTE_NAME attribute type " +
- s"must be a String or BinaryType")
+ s"must be a ${StringType.catalogString} or ${BinaryType.catalogString}")
}
schema.find(_.name == VALUE_ATTRIBUTE_NAME).getOrElse(
throw new AnalysisException(s"Required attribute '$VALUE_ATTRIBUTE_NAME' not found")
@@ -73,7 +73,7 @@ private[kafka010] object KafkaWriter extends Logging {
case StringType | BinaryType => // good
case _ =>
throw new AnalysisException(s"$VALUE_ATTRIBUTE_NAME attribute type " +
- s"must be a String or BinaryType")
+ s"must be a ${StringType.catalogString} or ${BinaryType.catalogString}")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSinkSuite.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSinkSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSinkSuite.scala
index ddfc0c1..0e1492a 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSinkSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSinkSuite.scala
@@ -314,7 +314,7 @@ class KafkaContinuousSinkSuite extends KafkaContinuousTest {
writer.stop()
}
assert(ex.getMessage.toLowerCase(Locale.ROOT).contains(
- "value attribute type must be a string or binarytype"))
+ "value attribute type must be a string or binary"))
try {
/* key field wrong type */
@@ -330,7 +330,7 @@ class KafkaContinuousSinkSuite extends KafkaContinuousTest {
writer.stop()
}
assert(ex.getMessage.toLowerCase(Locale.ROOT).contains(
- "key attribute type must be a string or binarytype"))
+ "key attribute type must be a string or binary"))
}
test("streaming - write to non-existing topic") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
index 7079ac6..70ffd7d 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
@@ -303,7 +303,7 @@ class KafkaSinkSuite extends StreamTest with SharedSQLContext {
writer.stop()
}
assert(ex.getMessage.toLowerCase(Locale.ROOT).contains(
- "value attribute type must be a string or binarytype"))
+ "value attribute type must be a string or binary"))
try {
ex = intercept[StreamingQueryException] {
@@ -318,7 +318,7 @@ class KafkaSinkSuite extends StreamTest with SharedSQLContext {
writer.stop()
}
assert(ex.getMessage.toLowerCase(Locale.ROOT).contains(
- "key attribute type must be a string or binarytype"))
+ "key attribute type must be a string or binary"))
}
test("streaming - write to non-existing topic") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
index 682787a..32d9815 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
@@ -69,7 +69,8 @@ class DCT @Since("1.5.0") (@Since("1.5.0") override val uid: String)
}
override protected def validateInputType(inputType: DataType): Unit = {
- require(inputType.isInstanceOf[VectorUDT], s"Input type must be VectorUDT but got $inputType.")
+ require(inputType.isInstanceOf[VectorUDT],
+ s"Input type must be ${(new VectorUDT).catalogString} but got ${inputType.catalogString}.")
}
override protected def outputDataType: DataType = new VectorUDT
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
index d67e481..dc38ee3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
@@ -208,8 +208,9 @@ class FeatureHasher(@Since("2.3.0") override val uid: String) extends Transforme
require(dataType.isInstanceOf[NumericType] ||
dataType.isInstanceOf[StringType] ||
dataType.isInstanceOf[BooleanType],
- s"FeatureHasher requires columns to be of NumericType, BooleanType or StringType. " +
- s"Column $fieldName was $dataType")
+ s"FeatureHasher requires columns to be of ${NumericType.simpleString}, " +
+ s"${BooleanType.catalogString} or ${StringType.catalogString}. " +
+ s"Column $fieldName was ${dataType.catalogString}")
}
val attrGroup = new AttributeGroup($(outputCol), $(numFeatures))
SchemaUtils.appendColumn(schema, attrGroup.toStructField())
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
index db432b6..dbda5b8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
@@ -104,7 +104,7 @@ class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
override def transformSchema(schema: StructType): StructType = {
val inputType = schema($(inputCol)).dataType
require(inputType.isInstanceOf[ArrayType],
- s"The input column must be ArrayType, but got $inputType.")
+ s"The input column must be ${ArrayType.simpleString}, but got ${inputType.catalogString}.")
val attrGroup = new AttributeGroup($(outputCol), $(numFeatures))
SchemaUtils.appendColumn(schema, attrGroup.toStructField())
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/Interaction.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Interaction.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Interaction.scala
index 4ff1d0e..611f1b6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Interaction.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Interaction.scala
@@ -261,7 +261,8 @@ private[ml] class FeatureEncoder(numFeatures: Array[Int]) extends Serializable {
*/
def foreachNonzeroOutput(value: Any, f: (Int, Double) => Unit): Unit = value match {
case d: Double =>
- assert(numFeatures.length == 1, "DoubleType columns should only contain one feature.")
+ assert(numFeatures.length == 1,
+ s"${DoubleType.catalogString} columns should only contain one feature.")
val numOutputCols = numFeatures.head
if (numOutputCols > 1) {
assert(
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
index c8760f9..e0772d5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
@@ -65,7 +65,8 @@ class NGram @Since("1.5.0") (@Since("1.5.0") override val uid: String)
override protected def validateInputType(inputType: DataType): Unit = {
require(inputType.sameType(ArrayType(StringType)),
- s"Input type must be ArrayType(StringType) but got $inputType.")
+ s"Input type must be ${ArrayType(StringType).catalogString} but got " +
+ inputType.catalogString)
}
override protected def outputDataType: DataType = new ArrayType(StringType, false)
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
index 5ab6c2d..27e4869 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
@@ -85,7 +85,8 @@ class OneHotEncoder @Since("1.4.0") (@Since("1.4.0") override val uid: String) e
val inputFields = schema.fields
require(schema(inputColName).dataType.isInstanceOf[NumericType],
- s"Input column must be of type NumericType but got ${schema(inputColName).dataType}")
+ s"Input column must be of type ${NumericType.simpleString} but got " +
+ schema(inputColName).dataType.catalogString)
require(!inputFields.exists(_.name == outputColName),
s"Output column $outputColName already exists.")
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index 55e595e..346e182 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -394,7 +394,7 @@ class RFormulaModel private[feature](
require(!columnNames.contains($(featuresCol)), "Features column already exists.")
require(
!columnNames.contains($(labelCol)) || schema($(labelCol)).dataType.isInstanceOf[NumericType],
- "Label column already exists and is not of type NumericType.")
+ s"Label column already exists and is not of type ${NumericType.simpleString}.")
}
@Since("2.0.0")
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
index 0f946dd..94640a5 100755
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
@@ -131,8 +131,8 @@ class StopWordsRemover @Since("1.5.0") (@Since("1.5.0") override val uid: String
@Since("1.5.0")
override def transformSchema(schema: StructType): StructType = {
val inputType = schema($(inputCol)).dataType
- require(inputType.sameType(ArrayType(StringType)),
- s"Input type must be ArrayType(StringType) but got $inputType.")
+ require(inputType.sameType(ArrayType(StringType)), "Input type must be " +
+ s"${ArrayType(StringType).catalogString} but got ${inputType.catalogString}.")
SchemaUtils.appendColumn(schema, $(outputCol), inputType, schema($(inputCol)).nullable)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
index cfaf6c0..aede1f8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
@@ -40,7 +40,8 @@ class Tokenizer @Since("1.4.0") (@Since("1.4.0") override val uid: String)
}
override protected def validateInputType(inputType: DataType): Unit = {
- require(inputType == StringType, s"Input type must be string type but got $inputType.")
+ require(inputType == StringType,
+ s"Input type must be ${StringType.catalogString} type but got ${inputType.catalogString}.")
}
override protected def outputDataType: DataType = new ArrayType(StringType, true)
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
index 4061154..57e23d5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
@@ -162,7 +162,7 @@ class VectorAssembler @Since("1.4.0") (@Since("1.4.0") override val uid: String)
schema(name).dataType match {
case _: NumericType | BooleanType => None
case t if t.isInstanceOf[VectorUDT] => None
- case other => Some(s"Data type $other of column $name is not supported.")
+ case other => Some(s"Data type ${other.catalogString} of column $name is not supported.")
}
}
if (incorrectColumns.nonEmpty) {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index d7fbe28..9d664b6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -106,7 +106,7 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
protected def validateAndTransformSchema(schema: StructType): StructType = {
val inputType = schema($(itemsCol)).dataType
require(inputType.isInstanceOf[ArrayType],
- s"The input column must be ArrayType, but got $inputType.")
+ s"The input column must be ${ArrayType.simpleString}, but got ${inputType.catalogString}.")
SchemaUtils.appendColumn(schema, $(predictionCol), schema($(itemsCol)).dataType)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala
index d9a3f85..c3894eb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala
@@ -41,7 +41,8 @@ private[spark] object SchemaUtils {
val actualDataType = schema(colName).dataType
val message = if (msg != null && msg.trim.length > 0) " " + msg else ""
require(actualDataType.equals(dataType),
- s"Column $colName must be of type $dataType but was actually $actualDataType.$message")
+ s"Column $colName must be of type ${dataType.catalogString} but was actually " +
+ s"${actualDataType.catalogString}.$message")
}
/**
@@ -58,7 +59,8 @@ private[spark] object SchemaUtils {
val message = if (msg != null && msg.trim.length > 0) " " + msg else ""
require(dataTypes.exists(actualDataType.equals),
s"Column $colName must be of type equal to one of the following types: " +
- s"${dataTypes.mkString("[", ", ", "]")} but was actually of type $actualDataType.$message")
+ s"${dataTypes.map(_.catalogString).mkString("[", ", ", "]")} but was actually of type " +
+ s"${actualDataType.catalogString}.$message")
}
/**
@@ -71,8 +73,9 @@ private[spark] object SchemaUtils {
msg: String = ""): Unit = {
val actualDataType = schema(colName).dataType
val message = if (msg != null && msg.trim.length > 0) " " + msg else ""
- require(actualDataType.isInstanceOf[NumericType], s"Column $colName must be of type " +
- s"NumericType but was actually of type $actualDataType.$message")
+ require(actualDataType.isInstanceOf[NumericType],
+ s"Column $colName must be of type ${NumericType.simpleString} but was actually of type " +
+ s"${actualDataType.catalogString}.$message")
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
index ede2847..2b0909a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluatorSuite.scala
@@ -67,8 +67,8 @@ class BinaryClassificationEvaluatorSuite
evaluator.evaluate(stringDF)
}
assert(thrown.getMessage.replace("\n", "") contains "Column rawPrediction must be of type " +
- "equal to one of the following types: [DoubleType, ")
- assert(thrown.getMessage.replace("\n", "") contains "but was actually of type StringType.")
+ "equal to one of the following types: [double, ")
+ assert(thrown.getMessage.replace("\n", "") contains "but was actually of type string.")
}
test("should support all NumericType labels and not support other types") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
index a250331..0de6528 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
@@ -105,7 +105,7 @@ class RFormulaSuite extends MLTest with DefaultReadWriteTest {
testTransformerByInterceptingException[(Int, Boolean)](
original,
model,
- "Label column already exists and is not of type NumericType.",
+ "Label column already exists and is not of type numeric.",
"x")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
index 91fb24a..ed15a1d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
@@ -99,9 +99,9 @@ class VectorAssemblerSuite
assembler.transform(df)
}
assert(thrown.getMessage contains
- "Data type StringType of column a is not supported.\n" +
- "Data type StringType of column b is not supported.\n" +
- "Data type StringType of column c is not supported.")
+ "Data type string of column a is not supported.\n" +
+ "Data type string of column b is not supported.\n" +
+ "Data type string of column c is not supported.")
}
test("ML attributes") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index e3dfe2f..65bee4e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -612,7 +612,7 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
estimator.fit(strDF)
}
assert(thrown.getMessage.contains(
- s"$column must be of type NumericType but was actually of type StringType"))
+ s"$column must be of type numeric but was actually of type string"))
}
private class NumericTypeWithEncoder[A](val numericType: NumericType)
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
index 4e4ff71..6cc73e0 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/AFTSurvivalRegressionSuite.scala
@@ -385,7 +385,7 @@ class AFTSurvivalRegressionSuite extends MLTest with DefaultReadWriteTest {
aft.fit(dfWithStringCensors)
}
assert(thrown.getMessage.contains(
- "Column censor must be of type NumericType but was actually of type StringType"))
+ "Column censor must be of type numeric but was actually of type string"))
}
test("numerical stability of standardization") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/mllib/src/test/scala/org/apache/spark/ml/util/MLTestingUtils.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/MLTestingUtils.scala b/mllib/src/test/scala/org/apache/spark/ml/util/MLTestingUtils.scala
index 5e72b4d..91a8b14 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/MLTestingUtils.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/MLTestingUtils.scala
@@ -74,7 +74,7 @@ object MLTestingUtils extends SparkFunSuite {
estimator.fit(dfWithStringLabels)
}
assert(thrown.getMessage.contains(
- "Column label must be of type NumericType but was actually of type StringType"))
+ "Column label must be of type numeric but was actually of type string"))
estimator match {
case weighted: Estimator[M] with HasWeightCol =>
@@ -86,7 +86,7 @@ object MLTestingUtils extends SparkFunSuite {
weighted.fit(dfWithStringWeights)
}
assert(thrown.getMessage.contains(
- "Column weight must be of type NumericType but was actually of type StringType"))
+ "Column weight must be of type numeric but was actually of type string"))
case _ =>
}
}
@@ -104,7 +104,7 @@ object MLTestingUtils extends SparkFunSuite {
evaluator.evaluate(dfWithStringLabels)
}
assert(thrown.getMessage.contains(
- "Column label must be of type NumericType but was actually of type StringType"))
+ "Column label must be of type numeric but was actually of type string"))
}
def genClassifDFWithNumericLabelCol(
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 59c371e..7c5504d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2273,7 +2273,7 @@ class Analyzer(
}
expr
case other =>
- throw new AnalysisException("need an array field but got " + other.simpleString)
+ throw new AnalysisException("need an array field but got " + other.catalogString)
}
}
validateNestedTupleFields(result)
@@ -2282,8 +2282,8 @@ class Analyzer(
}
private def fail(schema: StructType, maxOrdinal: Int): Unit = {
- throw new AnalysisException(s"Try to map ${schema.simpleString} to Tuple${maxOrdinal + 1}, " +
- "but failed as the number of fields does not line up.")
+ throw new AnalysisException(s"Try to map ${schema.catalogString} to Tuple${maxOrdinal + 1}" +
+ ", but failed as the number of fields does not line up.")
}
/**
@@ -2362,7 +2362,7 @@ class Analyzer(
case e => e.sql
}
throw new AnalysisException(s"Cannot up cast $fromStr from " +
- s"${from.dataType.simpleString} to ${to.simpleString} as it may truncate\n" +
+ s"${from.dataType.catalogString} to ${to.catalogString} as it may truncate\n" +
"The type path of the target object is:\n" + walkedTypePath.mkString("", "\n", "\n") +
"You can either add an explicit cast to the input data or choose a higher precision " +
"type of the field in the target object")
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index af256b9..49fe625 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -67,7 +67,7 @@ trait CheckAnalysis extends PredicateHelper {
limitExpr.sql)
case e if e.dataType != IntegerType => failAnalysis(
s"The limit expression must be integer type, but got " +
- e.dataType.simpleString)
+ e.dataType.catalogString)
case e if e.eval().asInstanceOf[Int] < 0 => failAnalysis(
"The limit expression must be equal to or greater than 0, but got " +
e.eval().asInstanceOf[Int])
@@ -96,8 +96,8 @@ trait CheckAnalysis extends PredicateHelper {
}
case c: Cast if !c.resolved =>
- failAnalysis(
- s"invalid cast from ${c.child.dataType.simpleString} to ${c.dataType.simpleString}")
+ failAnalysis(s"invalid cast from ${c.child.dataType.catalogString} to " +
+ c.dataType.catalogString)
case g: Grouping =>
failAnalysis("grouping() can only be used with GroupingSets/Cube/Rollup")
@@ -144,12 +144,12 @@ trait CheckAnalysis extends PredicateHelper {
case _ =>
failAnalysis(
s"Event time must be defined on a window or a timestamp, but " +
- s"${etw.eventTime.name} is of type ${etw.eventTime.dataType.simpleString}")
+ s"${etw.eventTime.name} is of type ${etw.eventTime.dataType.catalogString}")
}
case f: Filter if f.condition.dataType != BooleanType =>
failAnalysis(
s"filter expression '${f.condition.sql}' " +
- s"of type ${f.condition.dataType.simpleString} is not a boolean.")
+ s"of type ${f.condition.dataType.catalogString} is not a boolean.")
case Filter(condition, _) if hasNullAwarePredicateWithinNot(condition) =>
failAnalysis("Null-aware predicate sub-queries cannot be used in nested " +
@@ -158,7 +158,7 @@ trait CheckAnalysis extends PredicateHelper {
case j @ Join(_, _, _, Some(condition)) if condition.dataType != BooleanType =>
failAnalysis(
s"join condition '${condition.sql}' " +
- s"of type ${condition.dataType.simpleString} is not a boolean.")
+ s"of type ${condition.dataType.catalogString} is not a boolean.")
case Aggregate(groupingExprs, aggregateExprs, child) =>
def isAggregateExpression(expr: Expression) = {
@@ -219,7 +219,7 @@ trait CheckAnalysis extends PredicateHelper {
if (!RowOrdering.isOrderable(expr.dataType)) {
failAnalysis(
s"expression ${expr.sql} cannot be used as a grouping expression " +
- s"because its data type ${expr.dataType.simpleString} is not an orderable " +
+ s"because its data type ${expr.dataType.catalogString} is not an orderable " +
s"data type.")
}
@@ -239,7 +239,7 @@ trait CheckAnalysis extends PredicateHelper {
orders.foreach { order =>
if (!RowOrdering.isOrderable(order.dataType)) {
failAnalysis(
- s"sorting is not supported for columns of type ${order.dataType.simpleString}")
+ s"sorting is not supported for columns of type ${order.dataType.catalogString}")
}
}
@@ -342,7 +342,7 @@ trait CheckAnalysis extends PredicateHelper {
val mapCol = mapColumnInSetOperation(o).get
failAnalysis("Cannot have map type columns in DataFrame which calls " +
s"set operations(intersect, except, etc.), but the type of column ${mapCol.name} " +
- "is " + mapCol.dataType.simpleString)
+ "is " + mapCol.dataType.catalogString)
case o if o.expressions.exists(!_.deterministic) &&
!o.isInstanceOf[Project] && !o.isInstanceOf[Filter] &&
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
index 2021608..23eb78f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
@@ -76,7 +76,8 @@ case class AliasViewChild(conf: SQLConf) extends Rule[LogicalPlan] with CastSupp
// Will throw an AnalysisException if the cast can't perform or might truncate.
if (Cast.mayTruncate(originAttr.dataType, attr.dataType)) {
throw new AnalysisException(s"Cannot up cast ${originAttr.sql} from " +
- s"${originAttr.dataType.simpleString} to ${attr.simpleString} as it may truncate\n")
+ s"${originAttr.dataType.catalogString} to ${attr.dataType.catalogString} as it " +
+ s"may truncate\n")
} else {
Alias(cast(originAttr, attr.dataType), attr.name)(exprId = attr.exprId,
qualifier = attr.qualifier, explicitMetadata = Some(attr.metadata))
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 7971ae6..ba4d131 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -202,7 +202,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
TypeCheckResult.TypeCheckSuccess
} else {
TypeCheckResult.TypeCheckFailure(
- s"cannot cast ${child.dataType.simpleString} to ${dataType.simpleString}")
+ s"cannot cast ${child.dataType.catalogString} to ${dataType.catalogString}")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
index 98f25a9..464566b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
@@ -44,7 +44,7 @@ trait ExpectsInputTypes extends Expression {
val mismatches = children.zip(inputTypes).zipWithIndex.collect {
case ((child, expected), idx) if !expected.acceptsType(child.dataType) =>
s"argument ${idx + 1} requires ${expected.simpleString} type, " +
- s"however, '${child.sql}' is of ${child.dataType.simpleString} type."
+ s"however, '${child.sql}' is of ${child.dataType.catalogString} type."
}
if (mismatches.isEmpty) {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index a69b804..dcb9c96 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -580,10 +580,10 @@ abstract class BinaryOperator extends BinaryExpression with ExpectsInputTypes {
// First check whether left and right have the same type, then check if the type is acceptable.
if (!left.dataType.sameType(right.dataType)) {
TypeCheckResult.TypeCheckFailure(s"differing types in '$sql' " +
- s"(${left.dataType.simpleString} and ${right.dataType.simpleString}).")
+ s"(${left.dataType.catalogString} and ${right.dataType.catalogString}).")
} else if (!inputType.acceptsType(left.dataType)) {
TypeCheckResult.TypeCheckFailure(s"'$sql' requires ${inputType.simpleString} type," +
- s" not ${left.dataType.simpleString}")
+ s" not ${left.dataType.catalogString}")
} else {
TypeCheckResult.TypeCheckSuccess
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 3e7ca88..4b09978 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -1048,8 +1048,9 @@ case class ScalaUDF(
lazy val udfErrorMessage = {
val funcCls = function.getClass.getSimpleName
- val inputTypes = children.map(_.dataType.simpleString).mkString(", ")
- s"Failed to execute user defined function($funcCls: ($inputTypes) => ${dataType.simpleString})"
+ val inputTypes = children.map(_.dataType.catalogString).mkString(", ")
+ val outputType = dataType.catalogString
+ s"Failed to execute user defined function($funcCls: ($inputTypes) => $outputType)"
}
override def eval(input: InternalRow): Any = {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
index 76a8811..536276b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
@@ -73,7 +73,7 @@ case class SortOrder(
if (RowOrdering.isOrderable(dataType)) {
TypeCheckResult.TypeCheckSuccess
} else {
- TypeCheckResult.TypeCheckFailure(s"cannot sort data type ${dataType.simpleString}")
+ TypeCheckResult.TypeCheckFailure(s"cannot sort data type ${dataType.catalogString}")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index f1bbbda..c790d87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -132,7 +132,7 @@ case class ApproximatePercentile(
case TimestampType => value.asInstanceOf[Long].toDouble
case n: NumericType => n.numeric.toDouble(value.asInstanceOf[n.InternalType])
case other: DataType =>
- throw new UnsupportedOperationException(s"Unexpected data type ${other.simpleString}")
+ throw new UnsupportedOperationException(s"Unexpected data type ${other.catalogString}")
}
buffer.add(doubleValue)
}
@@ -157,7 +157,7 @@ case class ApproximatePercentile(
case DoubleType => doubleResult
case _: DecimalType => doubleResult.map(Decimal(_))
case other: DataType =>
- throw new UnsupportedOperationException(s"Unexpected data type ${other.simpleString}")
+ throw new UnsupportedOperationException(s"Unexpected data type ${other.catalogString}")
}
if (result.length == 0) {
null
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 5594041..c827226 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -528,7 +528,7 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
TypeCheckResult.TypeCheckFailure(
s"The expressions should all have the same type," +
- s" got LEAST(${children.map(_.dataType.simpleString).mkString(", ")}).")
+ s" got LEAST(${children.map(_.dataType.catalogString).mkString(", ")}).")
} else {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
}
@@ -601,7 +601,7 @@ case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpress
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
TypeCheckResult.TypeCheckFailure(
s"The expressions should all have the same type," +
- s" got GREATEST(${children.map(_.dataType.simpleString).mkString(", ")}).")
+ s" got GREATEST(${children.map(_.dataType.catalogString).mkString(", ")}).")
} else {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 838c045..05500f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -596,7 +596,7 @@ class CodegenContext {
case NullType => "false"
case _ =>
throw new IllegalArgumentException(
- "cannot generate equality code for un-comparable type: " + dataType.simpleString)
+ "cannot generate equality code for un-comparable type: " + dataType.catalogString)
}
/**
@@ -683,7 +683,7 @@ class CodegenContext {
case udt: UserDefinedType[_] => genComp(udt.sqlType, c1, c2)
case _ =>
throw new IllegalArgumentException(
- "cannot generate compare code for un-comparable type: " + dataType.simpleString)
+ "cannot generate compare code for un-comparable type: " + dataType.catalogString)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index d60f4c3..9263541 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -64,7 +64,7 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
TypeCheckResult.TypeCheckSuccess
case _ => TypeCheckResult.TypeCheckFailure(s"input to function $prettyName should have " +
s"been two ${ArrayType.simpleString}s with same element type, but it's " +
- s"[${left.dataType.simpleString}, ${right.dataType.simpleString}]")
+ s"[${left.dataType.catalogString}, ${right.dataType.catalogString}]")
}
}
}
@@ -509,7 +509,7 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
if (children.exists(!_.dataType.isInstanceOf[MapType])) {
TypeCheckResult.TypeCheckFailure(
s"input to $funcName should all be of type map, but it's " +
- children.map(_.dataType.simpleString).mkString("[", ", ", "]"))
+ children.map(_.dataType.catalogString).mkString("[", ", ", "]"))
} else {
TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), funcName)
}
@@ -751,7 +751,7 @@ case class MapFromEntries(child: Expression) extends UnaryExpression {
override def checkInputDataTypes(): TypeCheckResult = dataTypeDetails match {
case Some(_) => TypeCheckResult.TypeCheckSuccess
case None => TypeCheckResult.TypeCheckFailure(s"'${child.sql}' is of " +
- s"${child.dataType.simpleString} type. $prettyName accepts only arrays of pair structs.")
+ s"${child.dataType.catalogString} type. $prettyName accepts only arrays of pair structs.")
}
override protected def nullSafeEval(input: Any): Any = {
@@ -1118,7 +1118,7 @@ case class SortArray(base: Expression, ascendingOrder: Expression)
"Sort order in second argument requires a boolean literal.")
}
case ArrayType(dt, _) =>
- val dtSimple = dt.simpleString
+ val dtSimple = dt.catalogString
TypeCheckResult.TypeCheckFailure(
s"$prettyName does not support sorting array of type $dtSimple which is not orderable")
case _ =>
@@ -1166,7 +1166,7 @@ case class ArraySort(child: Expression) extends UnaryExpression with ArraySortLi
case ArrayType(dt, _) if RowOrdering.isOrderable(dt) =>
TypeCheckResult.TypeCheckSuccess
case ArrayType(dt, _) =>
- val dtSimple = dt.simpleString
+ val dtSimple = dt.catalogString
TypeCheckResult.TypeCheckFailure(
s"$prettyName does not support sorting array of type $dtSimple which is not orderable")
case _ =>
@@ -2217,7 +2217,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
return TypeCheckResult.TypeCheckFailure(
s"input to function $prettyName should have been ${StringType.simpleString}," +
s" ${BinaryType.simpleString} or ${ArrayType.simpleString}, but it's " +
- childTypes.map(_.simpleString).mkString("[", ", ", "]"))
+ childTypes.map(_.catalogString).mkString("[", ", ", "]"))
}
TypeUtils.checkForSameTypeInputExpr(childTypes, s"function $prettyName")
}
@@ -2424,7 +2424,7 @@ case class Flatten(child: Expression) extends UnaryExpression {
case _ =>
TypeCheckResult.TypeCheckFailure(
s"The argument should be an array of arrays, " +
- s"but '${child.sql}' is of ${child.dataType.simpleString} type."
+ s"but '${child.sql}' is of ${child.dataType.catalogString} type."
)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index a43de02..077a6dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -183,11 +183,11 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
} else if (!TypeCoercion.haveSameType(keys.map(_.dataType))) {
TypeCheckResult.TypeCheckFailure(
"The given keys of function map should all be the same type, but they are " +
- keys.map(_.dataType.simpleString).mkString("[", ", ", "]"))
+ keys.map(_.dataType.catalogString).mkString("[", ", ", "]"))
} else if (!TypeCoercion.haveSameType(values.map(_.dataType))) {
TypeCheckResult.TypeCheckFailure(
"The given values of function map should all be the same type, but they are " +
- values.map(_.dataType.simpleString).mkString("[", ", ", "]"))
+ values.map(_.dataType.catalogString).mkString("[", ", ", "]"))
} else {
TypeCheckResult.TypeCheckSuccess
}
@@ -388,8 +388,8 @@ trait CreateNamedStructLike extends Expression {
val invalidNames = nameExprs.filterNot(e => e.foldable && e.dataType == StringType)
if (invalidNames.nonEmpty) {
TypeCheckResult.TypeCheckFailure(
- "Only foldable StringType expressions are allowed to appear at odd position, got:" +
- s" ${invalidNames.mkString(",")}")
+ s"Only foldable ${StringType.catalogString} expressions are allowed to appear at odd" +
+ s" position, got: ${invalidNames.mkString(",")}")
} else if (!names.contains(null)) {
TypeCheckResult.TypeCheckSuccess
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 99671d5..8994eef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -68,7 +68,7 @@ object ExtractValue {
case StructType(_) =>
s"Field name should be String Literal, but it's $extraction"
case other =>
- s"Can't extract value from $child: need struct type but got ${other.simpleString}"
+ s"Can't extract value from $child: need struct type but got ${other.catalogString}"
}
throw new AnalysisException(errorMsg)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
index 3b597e8..bed581a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
@@ -47,10 +47,10 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
if (predicate.dataType != BooleanType) {
TypeCheckResult.TypeCheckFailure(
"type of predicate expression in If should be boolean, " +
- s"not ${predicate.dataType.simpleString}")
+ s"not ${predicate.dataType.catalogString}")
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
TypeCheckResult.TypeCheckFailure(s"differing types in '$sql' " +
- s"(${trueValue.dataType.simpleString} and ${falseValue.dataType.simpleString}).")
+ s"(${trueValue.dataType.catalogString} and ${falseValue.dataType.catalogString}).")
} else {
TypeCheckResult.TypeCheckSuccess
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index b7c52f1..b6e0d36 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -156,8 +156,8 @@ case class Stack(children: Seq[Expression]) extends Generator {
val j = (i - 1) % numFields
if (children(i).dataType != elementSchema.fields(j).dataType) {
return TypeCheckResult.TypeCheckFailure(
- s"Argument ${j + 1} (${elementSchema.fields(j).dataType.simpleString}) != " +
- s"Argument $i (${children(i).dataType.simpleString})")
+ s"Argument ${j + 1} (${elementSchema.fields(j).dataType.catalogString}) != " +
+ s"Argument $i (${children(i).dataType.catalogString})")
}
}
TypeCheckResult.TypeCheckSuccess
@@ -251,7 +251,7 @@ abstract class ExplodeBase extends UnaryExpression with CollectionGenerator with
case _ =>
TypeCheckResult.TypeCheckFailure(
"input to function explode should be array or map type, " +
- s"not ${child.dataType.simpleString}")
+ s"not ${child.dataType.catalogString}")
}
// hive-compatible default alias for explode function ("col" for array, "key", "value" for map)
@@ -381,7 +381,7 @@ case class Inline(child: Expression) extends UnaryExpression with CollectionGene
case _ =>
TypeCheckResult.TypeCheckFailure(
s"input to function $prettyName should be array of struct type, " +
- s"not ${child.dataType.simpleString}")
+ s"not ${child.dataType.catalogString}")
}
override def elementSchema: StructType = child.dataType match {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index 63943b1..abe8875 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -547,7 +547,7 @@ case class JsonToStructs(
case _: StructType | ArrayType(_: StructType, _) | _: MapType =>
super.checkInputDataTypes()
case _ => TypeCheckResult.TypeCheckFailure(
- s"Input schema ${nullableSchema.simpleString} must be a struct or an array of structs.")
+ s"Input schema ${nullableSchema.catalogString} must be a struct or an array of structs.")
}
@transient
@@ -729,7 +729,7 @@ case class StructsToJson(
TypeCheckResult.TypeCheckFailure(e.getMessage)
}
case _ => TypeCheckResult.TypeCheckFailure(
- s"Input type ${child.dataType.simpleString} must be a struct, array of structs or " +
+ s"Input type ${child.dataType.catalogString} must be a struct, array of structs or " +
"a map or array of map.")
}
@@ -790,7 +790,7 @@ object JsonExprUtils {
}
case m: CreateMap =>
throw new AnalysisException(
- s"A type of keys and values in map() must be string, but got ${m.dataType}")
+ s"A type of keys and values in map() must be string, but got ${m.dataType.catalogString}")
case _ =>
throw new AnalysisException("Must use a map() function for options")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 2bf4203..3189e68 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -1727,7 +1727,7 @@ case class ValidateExternalType(child: Expression, expected: DataType)
override val dataType: DataType = RowEncoder.externalDataTypeForInput(expected)
- private val errMsg = s" is not a valid external type for schema of ${expected.simpleString}"
+ private val errMsg = s" is not a valid external type for schema of ${expected.catalogString}"
private lazy val checkType: (Any) => Boolean = expected match {
case _: DecimalType =>
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index f54103c..699601e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -205,7 +205,7 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate {
}
case _ =>
TypeCheckResult.TypeCheckFailure(s"Arguments must be same type but were: " +
- s"${value.dataType.simpleString} != ${mismatchOpt.get.dataType.simpleString}")
+ s"${value.dataType.catalogString} != ${mismatchOpt.get.dataType.catalogString}")
}
} else {
TypeUtils.checkForOrderingExpr(value.dataType, s"function $prettyName")
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index bedad7d..1838b9f 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -222,12 +222,13 @@ case class Elt(children: Seq[Expression]) extends Expression {
val (indexType, inputTypes) = (indexExpr.dataType, inputExprs.map(_.dataType))
if (indexType != IntegerType) {
return TypeCheckResult.TypeCheckFailure(s"first input to function $prettyName should " +
- s"have IntegerType, but it's $indexType")
+ s"have ${IntegerType.catalogString}, but it's ${indexType.catalogString}")
}
if (inputTypes.exists(tpe => !Seq(StringType, BinaryType).contains(tpe))) {
return TypeCheckResult.TypeCheckFailure(
- s"input to function $prettyName should have StringType or BinaryType, but it's " +
- inputTypes.map(_.simpleString).mkString("[", ", ", "]"))
+ s"input to function $prettyName should have ${StringType.catalogString} or " +
+ s"${BinaryType.catalogString}, but it's " +
+ inputTypes.map(_.catalogString).mkString("[", ", ", "]"))
}
TypeUtils.checkForSameTypeInputExpr(inputTypes, s"function $prettyName")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index f957aaa..53c6f01 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -70,9 +70,9 @@ case class WindowSpecDefinition(
case f: SpecifiedWindowFrame if f.frameType == RangeFrame && f.isValueBound &&
!isValidFrameType(f.valueBoundary.head.dataType) =>
TypeCheckFailure(
- s"The data type '${orderSpec.head.dataType.simpleString}' used in the order " +
+ s"The data type '${orderSpec.head.dataType.catalogString}' used in the order " +
"specification does not match the data type " +
- s"'${f.valueBoundary.head.dataType.simpleString}' which is used in the range frame.")
+ s"'${f.valueBoundary.head.dataType.catalogString}' which is used in the range frame.")
case _ => TypeCheckSuccess
}
}
@@ -251,7 +251,7 @@ case class SpecifiedWindowFrame(
TypeCheckFailure(s"Window frame $location bound '$e' is not a literal.")
case e: Expression if !frameType.inputType.acceptsType(e.dataType) =>
TypeCheckFailure(
- s"The data type of the $location bound '${e.dataType.simpleString}' does not match " +
+ s"The data type of the $location bound '${e.dataType.catalogString}' does not match " +
s"the expected data type '${frameType.inputType.simpleString}'.")
case _ => TypeCheckSuccess
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index 9c413de..7389477 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -45,14 +45,14 @@ private[sql] class JacksonGenerator(
// `JackGenerator` can only be initialized with a `StructType` or a `MapType`.
require(dataType.isInstanceOf[StructType] || dataType.isInstanceOf[MapType],
- "JacksonGenerator only supports to be initialized with a StructType " +
- s"or MapType but got ${dataType.simpleString}")
+ s"JacksonGenerator only supports to be initialized with a ${StructType.simpleString} " +
+ s"or ${MapType.simpleString} but got ${dataType.catalogString}")
// `ValueWriter`s for all fields of the schema
private lazy val rootFieldWriters: Array[ValueWriter] = dataType match {
case st: StructType => st.map(_.dataType).map(makeWriter).toArray
case _ => throw new UnsupportedOperationException(
- s"Initial type ${dataType.simpleString} must be a struct")
+ s"Initial type ${dataType.catalogString} must be a struct")
}
// `ValueWriter` for array data storing rows of the schema.
@@ -70,7 +70,7 @@ private[sql] class JacksonGenerator(
private lazy val mapElementWriter: ValueWriter = dataType match {
case mt: MapType => makeWriter(mt.valueType)
case _ => throw new UnsupportedOperationException(
- s"Initial type ${dataType.simpleString} must be a map")
+ s"Initial type ${dataType.catalogString} must be a map")
}
private val gen = new JsonFactory().createGenerator(writer).setRootValueSeparator(null)
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index c3a4ca8..4d409ca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -143,7 +143,8 @@ class JacksonParser(
case "NaN" => Float.NaN
case "Infinity" => Float.PositiveInfinity
case "-Infinity" => Float.NegativeInfinity
- case other => throw new RuntimeException(s"Cannot parse $other as FloatType.")
+ case other => throw new RuntimeException(
+ s"Cannot parse $other as ${FloatType.catalogString}.")
}
}
@@ -158,7 +159,8 @@ class JacksonParser(
case "NaN" => Double.NaN
case "Infinity" => Double.PositiveInfinity
case "-Infinity" => Double.NegativeInfinity
- case other => throw new RuntimeException(s"Cannot parse $other as DoubleType.")
+ case other =>
+ throw new RuntimeException(s"Cannot parse $other as ${DoubleType.catalogString}.")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonUtils.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonUtils.scala
index 134d16e..f26b194 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonUtils.scala
@@ -52,7 +52,7 @@ object JacksonUtils {
case _ =>
throw new UnsupportedOperationException(
- s"Unable to convert column $name of type ${dataType.simpleString} to JSON.")
+ s"Unable to convert column $name of type ${dataType.catalogString} to JSON.")
}
schema.foreach(field => verifyType(field.name, field.dataType))
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 491ca00..5f70e06 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -294,8 +294,10 @@ private[sql] object JsonInferSchema {
// Both fields1 and fields2 should be sorted by name, since inferField performs sorting.
// Therefore, we can take advantage of the fact that we're merging sorted lists and skip
// building a hash map or performing additional sorting.
- assert(isSorted(fields1), s"StructType's fields were not sorted: ${fields1.toSeq}")
- assert(isSorted(fields2), s"StructType's fields were not sorted: ${fields2.toSeq}")
+ assert(isSorted(fields1),
+ s"${StructType.simpleString}'s fields were not sorted: ${fields1.toSeq}")
+ assert(isSorted(fields2),
+ s"${StructType.simpleString}'s fields were not sorted: ${fields2.toSeq}")
val newFields = new java.util.ArrayList[StructField]()
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
index b795abe..5214cdc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TypeUtils.scala
@@ -29,7 +29,7 @@ object TypeUtils {
if (dt.isInstanceOf[NumericType] || dt == NullType) {
TypeCheckResult.TypeCheckSuccess
} else {
- TypeCheckResult.TypeCheckFailure(s"$caller requires numeric types, not $dt")
+ TypeCheckResult.TypeCheckFailure(s"$caller requires numeric types, not ${dt.catalogString}")
}
}
@@ -37,7 +37,8 @@ object TypeUtils {
if (RowOrdering.isOrderable(dt)) {
TypeCheckResult.TypeCheckSuccess
} else {
- TypeCheckResult.TypeCheckFailure(s"$caller does not support ordering on type $dt")
+ TypeCheckResult.TypeCheckFailure(
+ s"$caller does not support ordering on type ${dt.catalogString}")
}
}
@@ -47,7 +48,7 @@ object TypeUtils {
} else {
return TypeCheckResult.TypeCheckFailure(
s"input to $caller should all be the same type, but it's " +
- types.map(_.simpleString).mkString("[", ", ", "]"))
+ types.map(_.catalogString).mkString("[", ", ", "]"))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
index 3041f44..c43cc74 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
@@ -145,7 +145,7 @@ abstract class NumericType extends AtomicType {
}
-private[sql] object NumericType extends AbstractDataType {
+private[spark] object NumericType extends AbstractDataType {
/**
* Enables matching against NumericType for expressions:
* {{{
@@ -155,11 +155,12 @@ private[sql] object NumericType extends AbstractDataType {
*/
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[NumericType]
- override private[sql] def defaultConcreteType: DataType = DoubleType
+ override private[spark] def defaultConcreteType: DataType = DoubleType
- override private[sql] def simpleString: String = "numeric"
+ override private[spark] def simpleString: String = "numeric"
- override private[sql] def acceptsType(other: DataType): Boolean = other.isInstanceOf[NumericType]
+ override private[spark] def acceptsType(other: DataType): Boolean =
+ other.isInstanceOf[NumericType]
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
index 38c4048..58c75b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
@@ -42,7 +42,7 @@ object ArrayType extends AbstractDataType {
other.isInstanceOf[ArrayType]
}
- override private[sql] def simpleString: String = "array"
+ override private[spark] def simpleString: String = "array"
}
/**
@@ -103,7 +103,8 @@ case class ArrayType(elementType: DataType, containsNull: Boolean) extends DataT
case a : ArrayType => a.interpretedOrdering.asInstanceOf[Ordering[Any]]
case s: StructType => s.interpretedOrdering.asInstanceOf[Ordering[Any]]
case other =>
- throw new IllegalArgumentException(s"Type $other does not support ordered operations")
+ throw new IllegalArgumentException(
+ s"Type ${other.catalogString} does not support ordered operations")
}
def compare(x: ArrayData, y: ArrayData): Int = {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index dbf51c3..f780ffd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -48,7 +48,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
}
if (precision > DecimalType.MAX_PRECISION) {
- throw new AnalysisException(s"DecimalType can only support precision up to 38")
+ throw new AnalysisException(
+ s"${DecimalType.simpleString} can only support precision up to ${DecimalType.MAX_PRECISION}")
}
// default constructor for Java
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
index 2d49fe0..203e85e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
@@ -24,7 +24,8 @@ import org.apache.spark.annotation.InterfaceStability
@InterfaceStability.Evolving
object ObjectType extends AbstractDataType {
override private[sql] def defaultConcreteType: DataType =
- throw new UnsupportedOperationException("null literals can't be casted to ObjectType")
+ throw new UnsupportedOperationException(
+ s"null literals can't be casted to ${ObjectType.simpleString}")
override private[sql] def acceptsType(other: DataType): Boolean = other match {
case ObjectType(_) => true
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index 362676b..b13e95f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -426,7 +426,7 @@ object StructType extends AbstractDataType {
private[sql] def fromString(raw: String): StructType = {
Try(DataType.fromJson(raw)).getOrElse(LegacyTypeStringParser.parse(raw)) match {
case t: StructType => t
- case _ => throw new RuntimeException(s"Failed parsing StructType: $raw")
+ case _ => throw new RuntimeException(s"Failed parsing ${StructType.simpleString}: $raw")
}
}
@@ -528,7 +528,8 @@ object StructType extends AbstractDataType {
leftType
case _ =>
- throw new SparkException(s"Failed to merge incompatible data types $left and $right")
+ throw new SparkException(s"Failed to merge incompatible data types ${left.catalogString}" +
+ s" and ${right.catalogString}")
}
private[sql] def fieldsMap(fields: Array[StructField]): Map[String, StructField] = {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 0ce94d3..f4cfed4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -521,7 +521,7 @@ class AnalysisErrorSuite extends AnalysisTest {
right,
joinType = Cross,
condition = Some('b === 'd))
- assertAnalysisError(plan2, "EqualTo does not support ordering on type MapType" :: Nil)
+ assertAnalysisError(plan2, "EqualTo does not support ordering on type map" :: Nil)
}
test("PredicateSubQuery is used outside of a filter") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 36714bd..8eec148 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -109,17 +109,17 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
assertErrorForDifferingTypes(GreaterThan('intField, 'booleanField))
assertErrorForDifferingTypes(GreaterThanOrEqual('intField, 'booleanField))
- assertError(EqualTo('mapField, 'mapField), "EqualTo does not support ordering on type MapType")
+ assertError(EqualTo('mapField, 'mapField), "EqualTo does not support ordering on type map")
assertError(EqualNullSafe('mapField, 'mapField),
- "EqualNullSafe does not support ordering on type MapType")
+ "EqualNullSafe does not support ordering on type map")
assertError(LessThan('mapField, 'mapField),
- "LessThan does not support ordering on type MapType")
+ "LessThan does not support ordering on type map")
assertError(LessThanOrEqual('mapField, 'mapField),
- "LessThanOrEqual does not support ordering on type MapType")
+ "LessThanOrEqual does not support ordering on type map")
assertError(GreaterThan('mapField, 'mapField),
- "GreaterThan does not support ordering on type MapType")
+ "GreaterThan does not support ordering on type map")
assertError(GreaterThanOrEqual('mapField, 'mapField),
- "GreaterThanOrEqual does not support ordering on type MapType")
+ "GreaterThanOrEqual does not support ordering on type map")
assertError(If('intField, 'stringField, 'stringField),
"type of predicate expression in If should be boolean")
@@ -169,10 +169,10 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
CreateNamedStruct(Seq("a", "b", 2.0)), "even number of arguments")
assertError(
CreateNamedStruct(Seq(1, "a", "b", 2.0)),
- "Only foldable StringType expressions are allowed to appear at odd position")
+ "Only foldable string expressions are allowed to appear at odd position")
assertError(
CreateNamedStruct(Seq('a.string.at(0), "a", "b", 2.0)),
- "Only foldable StringType expressions are allowed to appear at odd position")
+ "Only foldable string expressions are allowed to appear at odd position")
assertError(
CreateNamedStruct(Seq(Literal.create(null, StringType), "a")),
"Field name should not be null")
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index cb8a1fe..b4d422d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -469,7 +469,7 @@ class ExpressionParserSuite extends PlanTest {
Literal(BigDecimal("90912830918230182310293801923652346786").underlying()))
assertEqual("123.0E-28BD", Literal(BigDecimal("123.0E-28").underlying()))
assertEqual("123.08BD", Literal(BigDecimal("123.08").underlying()))
- intercept("1.20E-38BD", "DecimalType can only support precision up to 38")
+ intercept("1.20E-38BD", "decimal can only support precision up to 38")
}
test("strings") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a5925c16/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index 5a86f40..fccd057 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -154,7 +154,7 @@ class DataTypeSuite extends SparkFunSuite {
left.merge(right)
}.getMessage
assert(message.equals("Failed to merge fields 'b' and 'b'. " +
- "Failed to merge incompatible data types FloatType and LongType"))
+ "Failed to merge incompatible data types float and bigint"))
}
test("existsRecursively") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org