You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/03/26 00:08:08 UTC
[spark] branch branch-2.4 updated: [SPARK-26961][CORE] Enable
parallel classloading capability
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new c7fd233 [SPARK-26961][CORE] Enable parallel classloading capability
c7fd233 is described below
commit c7fd233e906f5df28ceb15aeadcd2d7267d04da5
Author: Ajith <aj...@gmail.com>
AuthorDate: Mon Mar 25 19:07:30 2019 -0500
[SPARK-26961][CORE] Enable parallel classloading capability
## What changes were proposed in this pull request?
As per https://docs.oracle.com/javase/8/docs/api/java/lang/ClassLoader.html
``Class loaders that support concurrent loading of classes are known as parallel capable class loaders and are required to register themselves at their class initialization time by invoking the ClassLoader.registerAsParallelCapable method. Note that the ClassLoader class is registered as parallel capable by default. However, its subclasses still need to register themselves if they are parallel capable. ``
i.e we can have finer class loading locks by registering classloaders as parallel capable. (Refer to deadlock due to macro lock https://issues.apache.org/jira/browse/SPARK-26961).
All the classloaders we have are wrapper of URLClassLoader which by itself is parallel capable.
But this cannot be achieved by scala code due to static registration Refer https://github.com/scala/bug/issues/11429
## How was this patch tested?
All Existing UT must pass
Closes #24126 from ajithme/driverlock.
Authored-by: Ajith <aj...@gmail.com>
Signed-off-by: Sean Owen <se...@databricks.com>
(cherry picked from commit b61dce23d2ee7ca95770bc7c390029aae8c65f7e)
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../spark/util/ChildFirstURLClassLoader.java | 68 ++++++++++++++++++++
.../apache/spark/util/MutableURLClassLoader.java} | 23 ++++---
.../org/apache/spark/util/ParentClassLoader.java} | 21 +++---
.../apache/spark/util/MutableURLClassLoader.scala | 75 ----------------------
.../internal/NonClosableMutableURLClassLoader.java | 24 +++----
.../apache/spark/sql/internal/SharedState.scala | 11 ----
6 files changed, 107 insertions(+), 115 deletions(-)
diff --git a/core/src/main/java/org/apache/spark/util/ChildFirstURLClassLoader.java b/core/src/main/java/org/apache/spark/util/ChildFirstURLClassLoader.java
new file mode 100644
index 0000000..57d9675
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/util/ChildFirstURLClassLoader.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+
+/**
+ * A mutable class loader that gives preference to its own URLs over the parent class loader
+ * when loading classes and resources.
+ */
+public class ChildFirstURLClassLoader extends MutableURLClassLoader {
+
+ static {
+ ClassLoader.registerAsParallelCapable();
+ }
+
+ private ParentClassLoader parent;
+
+ public ChildFirstURLClassLoader(URL[] urls, ClassLoader parent) {
+ super(urls, null);
+ this.parent = new ParentClassLoader(parent);
+ }
+
+ @Override
+ public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ try {
+ return super.loadClass(name, resolve);
+ } catch (ClassNotFoundException cnf) {
+ return parent.loadClass(name, resolve);
+ }
+ }
+
+ @Override
+ public Enumeration<URL> getResources(String name) throws IOException {
+ ArrayList<URL> urls = Collections.list(super.getResources(name));
+ urls.addAll(Collections.list(parent.getResources(name)));
+ return Collections.enumeration(urls);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ URL url = super.getResource(name);
+ if (url != null) {
+ return url;
+ } else {
+ return parent.getResource(name);
+ }
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala b/core/src/main/java/org/apache/spark/util/MutableURLClassLoader.java
similarity index 64%
copy from core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
copy to core/src/main/java/org/apache/spark/util/MutableURLClassLoader.java
index c9b7493..a7c775d 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++ b/core/src/main/java/org/apache/spark/util/MutableURLClassLoader.java
@@ -15,23 +15,26 @@
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.util;
+
+import java.net.URL;
+import java.net.URLClassLoader;
/**
- * A class loader which makes some protected methods in ClassLoader accessible.
+ * URL class loader that exposes the `addURL` method in URLClassLoader.
*/
-private[spark] class ParentClassLoader(parent: ClassLoader) extends ClassLoader(parent) {
+public class MutableURLClassLoader extends URLClassLoader {
- override def findClass(name: String): Class[_] = {
- super.findClass(name)
+ static {
+ ClassLoader.registerAsParallelCapable();
}
- override def loadClass(name: String): Class[_] = {
- super.loadClass(name)
+ public MutableURLClassLoader(URL[] urls, ClassLoader parent) {
+ super(urls, parent);
}
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- super.loadClass(name, resolve)
+ @Override
+ public void addURL(URL url) {
+ super.addURL(url);
}
-
}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala b/core/src/main/java/org/apache/spark/util/ParentClassLoader.java
similarity index 65%
copy from core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
copy to core/src/main/java/org/apache/spark/util/ParentClassLoader.java
index c9b7493..094005c 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++ b/core/src/main/java/org/apache/spark/util/ParentClassLoader.java
@@ -15,23 +15,28 @@
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.util;
/**
* A class loader which makes some protected methods in ClassLoader accessible.
*/
-private[spark] class ParentClassLoader(parent: ClassLoader) extends ClassLoader(parent) {
+public class ParentClassLoader extends ClassLoader {
- override def findClass(name: String): Class[_] = {
- super.findClass(name)
+ static {
+ ClassLoader.registerAsParallelCapable();
}
- override def loadClass(name: String): Class[_] = {
- super.loadClass(name)
+ public ParentClassLoader(ClassLoader parent) {
+ super(parent);
}
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- super.loadClass(name, resolve)
+ @Override
+ public Class<?> findClass(String name) throws ClassNotFoundException {
+ return super.findClass(name);
}
+ @Override
+ public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ return super.loadClass(name, resolve);
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala b/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
deleted file mode 100644
index 034826c..0000000
--- a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util
-
-import java.net.{URL, URLClassLoader}
-import java.util.Enumeration
-
-import scala.collection.JavaConverters._
-
-/**
- * URL class loader that exposes the `addURL` and `getURLs` methods in URLClassLoader.
- */
-private[spark] class MutableURLClassLoader(urls: Array[URL], parent: ClassLoader)
- extends URLClassLoader(urls, parent) {
-
- override def addURL(url: URL): Unit = {
- super.addURL(url)
- }
-
- override def getURLs(): Array[URL] = {
- super.getURLs()
- }
-
-}
-
-/**
- * A mutable class loader that gives preference to its own URLs over the parent class loader
- * when loading classes and resources.
- */
-private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
- extends MutableURLClassLoader(urls, null) {
-
- private val parentClassLoader = new ParentClassLoader(parent)
-
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- try {
- super.loadClass(name, resolve)
- } catch {
- case e: ClassNotFoundException =>
- parentClassLoader.loadClass(name, resolve)
- }
- }
-
- override def getResource(name: String): URL = {
- val url = super.findResource(name)
- val res = if (url != null) url else parentClassLoader.getResource(name)
- res
- }
-
- override def getResources(name: String): Enumeration[URL] = {
- val childUrls = super.findResources(name).asScala
- val parentUrls = parentClassLoader.getResources(name).asScala
- (childUrls ++ parentUrls).asJavaEnumeration
- }
-
- override def addURL(url: URL) {
- super.addURL(url)
- }
-
-}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala b/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
similarity index 63%
rename from core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
rename to sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
index c9b7493..db77d91 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++ b/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
@@ -15,23 +15,25 @@
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.sql.internal;
+
+import java.net.URL;
+
+import org.apache.spark.util.MutableURLClassLoader;
/**
- * A class loader which makes some protected methods in ClassLoader accessible.
+ * This class loader cannot be closed (its `close` method is a no-op).
*/
-private[spark] class ParentClassLoader(parent: ClassLoader) extends ClassLoader(parent) {
-
- override def findClass(name: String): Class[_] = {
- super.findClass(name)
- }
+public class NonClosableMutableURLClassLoader extends MutableURLClassLoader {
- override def loadClass(name: String): Class[_] = {
- super.loadClass(name)
+ static {
+ ClassLoader.registerAsParallelCapable();
}
- override def loadClass(name: String, resolve: Boolean): Class[_] = {
- super.loadClass(name, resolve)
+ public NonClosableMutableURLClassLoader(ClassLoader parent) {
+ super(new URL[]{}, parent);
}
+ @Override
+ public void close() {}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 5b6160e..4d2be13 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -193,14 +193,3 @@ object SharedState extends Logging {
}
}
}
-
-
-/**
- * URL class loader that exposes the `addURL` and `getURLs` methods in URLClassLoader.
- * This class loader cannot be closed (its `close` method is a no-op).
- */
-private[sql] class NonClosableMutableURLClassLoader(parent: ClassLoader)
- extends MutableURLClassLoader(Array.empty, parent) {
-
- override def close(): Unit = {}
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org