You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/11/26 04:25:33 UTC
spark git commit: [SPARK-18583][SQL] Fix nullability of InputFileName.
Repository: spark
Updated Branches:
refs/heads/master e2fb9fd36 -> a88329d45
[SPARK-18583][SQL] Fix nullability of InputFileName.
## What changes were proposed in this pull request?
The nullability of `InputFileName` should be `false`.
## How was this patch tested?
Existing tests.
Author: Takuya UESHIN <ue...@happy-camper.st>
Closes #16007 from ueshin/issues/SPARK-18583.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a88329d4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a88329d4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a88329d4
Branch: refs/heads/master
Commit: a88329d4553b40c45ebf9eacf229db7839d46769
Parents: e2fb9fd
Author: Takuya UESHIN <ue...@happy-camper.st>
Authored: Fri Nov 25 20:25:29 2016 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Nov 25 20:25:29 2016 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/rdd/InputFileNameHolder.scala | 10 +++++++++-
.../spark/sql/catalyst/expressions/InputFileName.scala | 2 +-
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a88329d4/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala b/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala
index f40d4c8..960c91a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala
@@ -22,6 +22,8 @@ import org.apache.spark.unsafe.types.UTF8String
/**
* This holds file names of the current Spark task. This is used in HadoopRDD,
* FileScanRDD, NewHadoopRDD and InputFileName function in Spark SQL.
+ *
+ * The returned value should never be null but empty string if it is unknown.
*/
private[spark] object InputFileNameHolder {
/**
@@ -32,9 +34,15 @@ private[spark] object InputFileNameHolder {
override protected def initialValue(): UTF8String = UTF8String.fromString("")
}
+ /**
+ * Returns the holding file name or empty string if it is unknown.
+ */
def getInputFileName(): UTF8String = inputFileName.get()
- private[spark] def setInputFileName(file: String) = inputFileName.set(UTF8String.fromString(file))
+ private[spark] def setInputFileName(file: String) = {
+ require(file != null, "The input file name cannot be null")
+ inputFileName.set(UTF8String.fromString(file))
+ }
private[spark] def unsetInputFileName(): Unit = inputFileName.remove()
http://git-wip-us.apache.org/repos/asf/spark/blob/a88329d4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala
index b7fb285..d412336 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala
@@ -30,7 +30,7 @@ import org.apache.spark.unsafe.types.UTF8String
usage = "_FUNC_() - Returns the name of the current file being read if available.")
case class InputFileName() extends LeafExpression with Nondeterministic {
- override def nullable: Boolean = true
+ override def nullable: Boolean = false
override def dataType: DataType = StringType
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org