[jira] [Created] (ZEPPELIN-2335) Setting for maxClassfileName_ will cause class not found exception

["wuyonghua (JIRA)" <ji...@apache.org>]

wuyonghua created ZEPPELIN-2335:
-----------------------------------

             Summary: Setting for maxClassfileName_ will cause class not found exception
                 Key: ZEPPELIN-2335
                 URL: https://issues.apache.org/jira/browse/ZEPPELIN-2335
             Project: Zeppelin
          Issue Type: Bug
          Components: Interpreters
            Reporter: wuyonghua


Code to run:
object Shell extends Serializable { // show
org.apache.spark.sql.catalyst.encoders.OuterScopes.addOuterScope(this)
case class Step4_Class(`id` : String,`factory` : String)
def main() {
    def _1002() : org.apache.spark.rdd.RDD[String] = {
        var retval : org.apache.spark.rdd.RDD[String] = null
        retval = sc.parallelize(Array(
            "id,factory",
            "0000,Nanjing"
        ))
        retval
    }
    val __1002 = _1002()
    def _1003(source1 : org.apache.spark.rdd.RDD[String]) : org.apache.spark.rdd.RDD[Step4_Class] = {
        val retval : org.apache.spark.rdd.RDD[Step4_Class] = source1 .filter(line => line != "id,factory") .mapPartitions(lines => 
        {
          val csvParser = new au.com.bytecode.opencsv.CSVParser(',')
          lines.flatMap {
            line => {
              try {
                val field = csvParser.parseLine(line)
                Seq(Step4_Class(field(0),field(1)))
              } catch {
                case e: Throwable => {
                  Seq()
                }
              }
            }
          }
        }, 
        true)
        retval
    }
    val __1003 = _1003(__1002)
    
    def _1004(source2 : org.apache.spark.rdd.RDD[Step4_Class]) : org.apache.spark.sql.DataFrame = {
        val __mapresult2=source2.toDF("__2id","__2factory")
        val retval=__mapresult2
        retval
    }
    val __1004 = _1004(__1003)
}
}
Shell.main()

Exception:
java.lang.ClassNotFoundException: no Java class corresponding to class $iw found
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.scala$reflect$runtime$JavaMirrors$JavaMirror$$anonfun$$noClass$1(JavaMirrors.scala:1204)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1242)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1218)
  at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
  at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
  at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
  at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.typeToJavaClass(JavaMirrors.scala:1297)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.runtimeClass(JavaMirrors.scala:192)
  at scala.reflect.runtime.JavaMirrors$JavaMirror.runtimeClass(JavaMirrors.scala:54)
  at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:50)
  at org.apache.spark.sql.Encoders$.product(Encoders.scala:274)
  at org.apache.spark.sql.SQLImplicits.newProductEncoder(SQLImplicits.scala:47)
  at $$$$44c822d8a716021c78a068729b15bb$$$$$Shell$._1002$1(:141)
  at $$$$44c822d8a716021c78a068729b15bb$$$$$Shell$.main(:161)
  ... 72 elided

Reason:
In such situation, case class Step4_Class has very long name(if not, pls add some imports before the code to make it longer).
SparkInterpreter.java sets maxClassfileName to 128, but scala has some places to create new setting, which use the default value 255.
This inconsistent will cause class not found, because some name will be encoded by MD5 to make it shorter than 120 (=128-8), while the other name is limited by 247(=255-8), thus they are not matched.
For example, the following code creates a new setting instead of using the global one.
class JavaUniverse extends InternalSymbolTable with JavaUniverseForce with ReflectSetup with RuntimeSymbolTable { self =>
              def picklerPhase = SomePhase
              def erasurePhase = SomePhase
              lazy val settings = new Settings

 




--
This message was sent by Atlassian JIRA
(v6.3.15#6346)