You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2018/09/19 22:16:24 UTC
spark git commit: [SPARK-22666][ML][FOLLOW-UP] Improve testcase to
tolerate different schema representation
Repository: spark
Updated Branches:
refs/heads/master cb1b55cf7 -> 6f681d429
[SPARK-22666][ML][FOLLOW-UP] Improve testcase to tolerate different schema representation
## What changes were proposed in this pull request?
Improve testcase "image datasource test: read non image" to tolerate different schema representation.
Because file:/path and file:///path are both valid URI-ifications so in some environment the testcase will fail.
## How was this patch tested?
Manual.
Closes #22449 from WeichenXu123/image_url.
Authored-by: WeichenXu <we...@databricks.com>
Signed-off-by: Xiangrui Meng <me...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6f681d42
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6f681d42
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6f681d42
Branch: refs/heads/master
Commit: 6f681d42964884d19bf22deb614550d712223117
Parents: cb1b55c
Author: WeichenXu <we...@databricks.com>
Authored: Wed Sep 19 15:16:20 2018 -0700
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Wed Sep 19 15:16:20 2018 -0700
----------------------------------------------------------------------
.../spark/ml/source/image/ImageFileFormatSuite.scala | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6f681d42/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
index 1a6a8d6..38e2513 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.ml.source.image
+import java.net.URI
import java.nio.file.Paths
import org.apache.spark.SparkFunSuite
@@ -58,8 +59,14 @@ class ImageFileFormatSuite extends SparkFunSuite with MLlibTestSparkContext {
.load(filePath)
assert(df2.count() === 1)
val result = df2.head()
- assert(result === invalidImageRow(
- Paths.get(filePath).toAbsolutePath().normalize().toUri().toString))
+
+ val resultOrigin = result.getStruct(0).getString(0)
+ // covert `origin` to `java.net.URI` object and then compare.
+ // because `file:/path` and `file:///path` are both valid URI-ifications
+ assert(new URI(resultOrigin) === Paths.get(filePath).toAbsolutePath().normalize().toUri())
+
+ // Compare other columns in the row to be the same with the `invalidImageRow`
+ assert(result === invalidImageRow(resultOrigin))
}
test("image datasource partition test") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org