You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/12/03 10:12:31 UTC
spark git commit: [SPARK-18685][TESTS] Fix URI and release resources
after opening in tests at ExecutorClassLoaderSuite
Repository: spark
Updated Branches:
refs/heads/master 553aac56b -> d1312fb7e
[SPARK-18685][TESTS] Fix URI and release resources after opening in tests at ExecutorClassLoaderSuite
## What changes were proposed in this pull request?
This PR fixes two problems as below:
- Close `BufferedSource` after `Source.fromInputStream(...)` to release resource and make the tests pass on Windows in `ExecutorClassLoaderSuite`
```
[info] Exception encountered when attempting to run a suite with class name: org.apache.spark.repl.ExecutorClassLoaderSuite *** ABORTED *** (7 seconds, 333 milliseconds)
[info] java.io.IOException: Failed to delete: C:\projects\spark\target\tmp\spark-77b2f37b-6405-47c4-af1c-4a6a206511f2
[info] at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:1010)
[info] at org.apache.spark.repl.ExecutorClassLoaderSuite.afterAll(ExecutorClassLoaderSuite.scala:76)
[info] at org.scalatest.BeforeAndAfterAll$class.afterAll(BeforeAndAfterAll.scala:213)
...
```
- Fix URI correctly so that related tests can be passed on Windows.
```
[info] - child first *** FAILED *** (78 milliseconds)
[info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b
[info] at java.net.URI$Parser.fail(URI.java:2848)
[info] at java.net.URI$Parser.parseAuthority(URI.java:3186)
...
[info] - parent first *** FAILED *** (15 milliseconds)
[info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b
[info] at java.net.URI$Parser.fail(URI.java:2848)
[info] at java.net.URI$Parser.parseAuthority(URI.java:3186)
...
[info] - child first can fall back *** FAILED *** (0 milliseconds)
[info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b
[info] at java.net.URI$Parser.fail(URI.java:2848)
[info] at java.net.URI$Parser.parseAuthority(URI.java:3186)
...
[info] - child first can fail *** FAILED *** (0 milliseconds)
[info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b
[info] at java.net.URI$Parser.fail(URI.java:2848)
[info] at java.net.URI$Parser.parseAuthority(URI.java:3186)
...
[info] - resource from parent *** FAILED *** (0 milliseconds)
[info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b
[info] at java.net.URI$Parser.fail(URI.java:2848)
[info] at java.net.URI$Parser.parseAuthority(URI.java:3186)
...
[info] - resources from parent *** FAILED *** (0 milliseconds)
[info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b
[info] at java.net.URI$Parser.fail(URI.java:2848)
[info] at java.net.URI$Parser.parseAuthority(URI.java:3186)
```
## How was this patch tested?
Manually tested via AppVeyor.
**Before**
https://ci.appveyor.com/project/spark-test/spark/build/102-rpel-ExecutorClassLoaderSuite
**After**
https://ci.appveyor.com/project/spark-test/spark/build/108-rpel-ExecutorClassLoaderSuite
Author: hyukjinkwon <gu...@gmail.com>
Closes #16116 from HyukjinKwon/close-after-open.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d1312fb7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d1312fb7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d1312fb7
Branch: refs/heads/master
Commit: d1312fb7edffd6e10c86f69ddfff05f8915856ac
Parents: 553aac5
Author: hyukjinkwon <gu...@gmail.com>
Authored: Sat Dec 3 10:12:28 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Sat Dec 3 10:12:28 2016 +0000
----------------------------------------------------------------------
.../spark/repl/ExecutorClassLoaderSuite.scala | 25 +++++++++++++-------
1 file changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d1312fb7/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
index 3d622d4..6d274bd 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -24,7 +24,6 @@ import java.nio.charset.StandardCharsets
import java.nio.file.{Paths, StandardOpenOption}
import java.util
-import scala.concurrent.duration._
import scala.io.Source
import scala.language.implicitConversions
@@ -34,8 +33,6 @@ import org.mockito.Mockito._
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import org.scalatest.BeforeAndAfterAll
-import org.scalatest.concurrent.Interruptor
-import org.scalatest.concurrent.Timeouts._
import org.scalatest.mock.MockitoSugar
import org.apache.spark._
@@ -61,7 +58,7 @@ class ExecutorClassLoaderSuite
super.beforeAll()
tempDir1 = Utils.createTempDir()
tempDir2 = Utils.createTempDir()
- url1 = "file://" + tempDir1
+ url1 = tempDir1.toURI.toURL.toString
urls2 = List(tempDir2.toURI.toURL).toArray
childClassNames.foreach(TestUtils.createCompiledClass(_, tempDir1, "1"))
parentResourceNames.foreach { x =>
@@ -118,8 +115,14 @@ class ExecutorClassLoaderSuite
val resourceName: String = parentResourceNames.head
val is = classLoader.getResourceAsStream(resourceName)
assert(is != null, s"Resource $resourceName not found")
- val content = Source.fromInputStream(is, "UTF-8").getLines().next()
- assert(content.contains("resource"), "File doesn't contain 'resource'")
+
+ val bufferedSource = Source.fromInputStream(is, "UTF-8")
+ Utils.tryWithSafeFinally {
+ val content = bufferedSource.getLines().next()
+ assert(content.contains("resource"), "File doesn't contain 'resource'")
+ } {
+ bufferedSource.close()
+ }
}
test("resources from parent") {
@@ -128,8 +131,14 @@ class ExecutorClassLoaderSuite
val resourceName: String = parentResourceNames.head
val resources: util.Enumeration[URL] = classLoader.getResources(resourceName)
assert(resources.hasMoreElements, s"Resource $resourceName not found")
- val fileReader = Source.fromInputStream(resources.nextElement().openStream()).bufferedReader()
- assert(fileReader.readLine().contains("resource"), "File doesn't contain 'resource'")
+
+ val bufferedSource = Source.fromInputStream(resources.nextElement().openStream())
+ Utils.tryWithSafeFinally {
+ val fileReader = bufferedSource.bufferedReader()
+ assert(fileReader.readLine().contains("resource"), "File doesn't contain 'resource'")
+ } {
+ bufferedSource.close()
+ }
}
test("fetch classes using Spark's RpcEnv") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org