[GitHub] [spark] mridulm commented on a change in pull request #34158: [SPARK-33781][FOLLOW-UP] Support the case when user's classes need to register for Kryo serialization

[GitBox <gi...@apache.org>]

mridulm commented on a change in pull request #34158:
URL: https://github.com/apache/spark/pull/34158#discussion_r719682418



##########
File path: core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
##########
@@ -463,7 +463,7 @@ private[spark] object ShuffleBlockPusher {
 
   private val BLOCK_PUSHER_POOL: ExecutorService = {
     val conf = SparkEnv.get.conf
-    if (Utils.isPushBasedShuffleEnabled(conf)) {
+    if (Utils.isPushBasedShuffleEnabled(conf, isDriver = false)) {

Review comment:
       We can probably change this to `, isDriver = SparkContext.DRIVER_IDENTIFIER == SparkEnv.get.executorId` to be forward compatible

##########
File path: core/src/main/scala/org/apache/spark/util/Utils.scala
##########
@@ -2627,6 +2627,37 @@ private[spark] object Utils extends Logging {
     }
   }
 
+  // Create an instance of the class with the given name, possibly initializing it with our conf
+  def instantiateClass[T](className: String, conf: SparkConf, isDriver: Boolean): T = {
+    val cls = classForName(className)
+    // Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just
+    // SparkConf, then one taking no arguments
+    try {
+      cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE)
+        .newInstance(conf, java.lang.Boolean.valueOf(isDriver))
+        .asInstanceOf[T]
+    } catch {
+      case _: NoSuchMethodException =>
+        try {
+          cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T]
+        } catch {
+          case _: NoSuchMethodException =>
+            cls.getConstructor().newInstance().asInstanceOf[T]
+        }
+    }
+  }
+
+  // Create an instance of the class named by the given SparkConf property
+  // if the property is not set, possibly initializing it with our conf
+  def instantiateClassFromConf[T](propertyName: ConfigEntry[String],
+      conf: SparkConf, isDriver: Boolean): T = {
+    instantiateClass[T](conf.get(propertyName), conf, isDriver)
+  }
+
+  def instantiateSerializer(conf: SparkConf, isDriver: Boolean): Serializer = {
+    instantiateClassFromConf[Serializer](SERIALIZER, conf, isDriver)
+  }

Review comment:
       Remove this method and inline this ?

##########
File path: core/src/main/scala/org/apache/spark/util/Utils.scala
##########
@@ -2627,6 +2627,37 @@ private[spark] object Utils extends Logging {
     }
   }
 
+  // Create an instance of the class with the given name, possibly initializing it with our conf
+  def instantiateClass[T](className: String, conf: SparkConf, isDriver: Boolean): T = {

Review comment:
       Make this private ?




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org