You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "chong (Jira)" <ji...@apache.org> on 2022/03/31 11:14:00 UTC
[jira] [Created] (SPARK-38714) Interval multiplication error
chong created SPARK-38714:
-----------------------------
Summary: Interval multiplication error
Key: SPARK-38714
URL: https://issues.apache.org/jira/browse/SPARK-38714
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 3.3.0
Environment: branch-3.3, Java 8
Reporter: chong
Code gen have error when multipling interval by a decimal.
$SPARK_HOME/bin/spark-shell
import org.apache.spark.sql.Row
import java.time.Duration
import java.time.Period
import org.apache.spark.sql.types._
val data = Seq(Row(new java.math.BigDecimal("123456789.11")))
val schema = StructType(Seq(
StructField("c1", DecimalType(9, 2)),
))
val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.selectExpr("interval '100' second * c1").show(false)
errors are:
java.lang.AssertionError: assertion failed:
Decimal$DecimalIsFractional
while compiling: <console>
during phase: globalPhase=terminal, enteringPhase=jvm
library version: version 2.12.15
compiler version: version 2.12.15
reconstructed args: -classpath -Yrepl-class-based -Yrepl-outdir /tmp/spark-83a0cda4-dd0b-472e-ad8b-a4b33b85f613/repl-06489815-5366-4aa0-9419-f01abda8d041
last tree to typer: TypeTree(class Byte)
tree position: line 6 of <console>
tree tpe: Byte
symbol: (final abstract) class Byte in package scala
symbol definition: final abstract class Byte extends (a ClassSymbol)
symbol package: scala
symbol owners: class Byte
call site: constructor $eval in object $eval in package $line21
== Source file context for tree position ==
3
4 object $eval {
5 lazy val $result = $line21.$read.INSTANCE.$iw.$iw.$iw.$iw.$iw.$iw.$iw.$iw.$iw.$iw.res0
6 lazy val $print: _root_.java.lang.String = {
7 $line21.$read.INSTANCE.$iw.$iw.$iw.$iw.$iw.$iw.$iw.$iw.$iw.$iw
8
9 ""
at scala.reflect.internal.SymbolTable.throwAssertionError(SymbolTable.scala:185)
at scala.reflect.internal.Symbols$Symbol.completeInfo(Symbols.scala:1525)
at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1514)
at scala.reflect.internal.Symbols$Symbol.flatOwnerInfo(Symbols.scala:2353)
at scala.reflect.internal.Symbols$ClassSymbol.companionModule0(Symbols.scala:3346)
at scala.reflect.internal.Symbols$ClassSymbol.companionModule(Symbols.scala:3348)
at scala.reflect.internal.Symbols$ModuleClassSymbol.sourceModule(Symbols.scala:3487)
at scala.reflect.internal.Symbols.$anonfun$forEachRelevantSymbols$1$adapted(Symbols.scala:3802)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
at scala.reflect.internal.Symbols.markFlagsCompleted(Symbols.scala:3799)
at scala.reflect.internal.Symbols.markFlagsCompleted$(Symbols.scala:3805)
at scala.reflect.internal.SymbolTable.markFlagsCompleted(SymbolTable.scala:28)
at scala.reflect.internal.pickling.UnPickler$Scan.finishSym$1(UnPickler.scala:324)
at scala.reflect.internal.pickling.UnPickler$Scan.readSymbol(UnPickler.scala:342)
at scala.reflect.internal.pickling.UnPickler$Scan.readSymbolRef(UnPickler.scala:645)
at scala.reflect.internal.pickling.UnPickler$Scan.readType(UnPickler.scala:413)
at scala.reflect.internal.pickling.UnPickler$Scan.$anonfun$readSymbol$10(UnPickler.scala:357)
at scala.reflect.internal.pickling.UnPickler$Scan.at(UnPickler.scala:188)
at scala.reflect.internal.pickling.UnPickler$Scan.readSymbol(UnPickler.scala:357)
at scala.reflect.internal.pickling.UnPickler$Scan.$anonfun$run$1(UnPickler.scala:96)
at scala.reflect.internal.pickling.UnPickler$Scan.run(UnPickler.scala:88)
at scala.reflect.internal.pickling.UnPickler.unpickle(UnPickler.scala:47)
at scala.tools.nsc.symtab.classfile.ClassfileParser.unpickleOrParseInnerClasses(ClassfileParser.scala:1186)
at scala.tools.nsc.symtab.classfile.ClassfileParser.parseClass(ClassfileParser.scala:468)
at scala.tools.nsc.symtab.classfile.ClassfileParser.$anonfun$parse$2(ClassfileParser.scala:161)
at scala.tools.nsc.symtab.classfile.ClassfileParser.$anonfun$parse$1(ClassfileParser.scala:147)
at scala.tools.nsc.symtab.classfile.ClassfileParser.parse(ClassfileParser.scala:130)
at scala.tools.nsc.symtab.SymbolLoaders$ClassfileLoader.doComplete(SymbolLoaders.scala:343)
at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:250)
at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.load(SymbolLoaders.scala:269)
at scala.reflect.internal.Symbols$Symbol.exists(Symbols.scala:1104)
at scala.reflect.internal.Symbols$Symbol.toOption(Symbols.scala:2609)
at scala.tools.nsc.interpreter.IMain.translateSimpleResource(IMain.scala:340)
at scala.tools.nsc.interpreter.IMain$TranslatingClassLoader.findAbstractFile(IMain.scala:354)
at scala.reflect.internal.util.AbstractFileClassLoader.findResource(AbstractFileClassLoader.scala:76)
at java.lang.ClassLoader.getResource(ClassLoader.java:1089)
at java.lang.ClassLoader.getResourceAsStream(ClassLoader.java:1300)
at scala.reflect.internal.util.RichClassLoader$.classAsStream$extension(ScalaClassLoader.scala:89)
at scala.reflect.internal.util.RichClassLoader$.classBytes$extension(ScalaClassLoader.scala:81)
at scala.reflect.internal.util.ScalaClassLoader.classBytes(ScalaClassLoader.scala:131)
at scala.reflect.internal.util.ScalaClassLoader.classBytes$(ScalaClassLoader.scala:131)
at scala.reflect.internal.util.AbstractFileClassLoader.classBytes(AbstractFileClassLoader.scala:41)
at scala.reflect.internal.util.AbstractFileClassLoader.findClass(AbstractFileClassLoader.scala:70)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at java.lang.ClassLoader.loadClass(ClassLoader.java:405)
at java.lang.ClassLoader.loadClass(ClassLoader.java:405)
at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.java:40)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:109)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at java.lang.ClassLoader.loadClass(ClassLoader.java:405)
at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.java:40)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89)
at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:317)
at org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8618)
at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8838)
at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8529)
at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8388)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6900)
at org.codehaus.janino.UnitCompiler.access$14600(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6518)
at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6515)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4429)
at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6515)
at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6511)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:4353)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6511)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4321)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6490)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9110)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7164)
at org.codehaus.janino.UnitCompiler.access$16200(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6538)
at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6511)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5286)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6511)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4321)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6490)
at org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9306)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9192)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9110)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5055)
at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4482)
at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4455)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5286)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4455)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5683)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3839)
at org.codehaus.janino.UnitCompiler.access$6100(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$13.visitAssignment(UnitCompiler.java:3799)
at org.codehaus.janino.UnitCompiler$13.visitAssignment(UnitCompiler.java:3779)
at org.codehaus.janino.Java$Assignment.accept(Java.java:4690)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3779)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2366)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1497)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1490)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:3064)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1490)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1573)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1559)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1496)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1490)
at org.codehaus.janino.Java$Block.accept(Java.java:2969)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1490)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2478)
at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1498)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1490)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:3140)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1490)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1573)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3420)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1362)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1335)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:807)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:975)
at org.codehaus.janino.UnitCompiler.access$700(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:392)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:384)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1445)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:384)
at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1312)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:833)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:410)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:389)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:384)
at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1594)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:384)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:362)
at org.codehaus.janino.UnitCompiler.access$000(UnitCompiler.java:226)
at org.codehaus.janino.UnitCompiler$1.visitCompilationUnit(UnitCompiler.java:336)
at org.codehaus.janino.UnitCompiler$1.visitCompilationUnit(UnitCompiler.java:333)
at org.codehaus.janino.Java$CompilationUnit.accept(Java.java:363)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:333)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:235)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:464)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:314)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:237)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:80)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1490)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1587)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1584)
at org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1437)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:378)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:331)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:34)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:1363)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:204)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:193)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:461)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:760)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:364)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:890)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:890)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:136)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:507)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:510)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
error: error while loading Decimal, class file '/home/chong/progs/sparks/spark-home/jars/spark-catalyst_2.12-3.3.0-SNAPSHOT.jar(org/apache/spark/sql/types/Decimal.class)' is broken
(class java.lang.RuntimeException/error reading Scala signature of Decimal.class: assertion failed:
Decimal$DecimalIsFractional
while compiling: <console>
during phase: globalPhase=terminal, enteringPhase=jvm
library version: version 2.12.15
compiler version: version 2.12.15
reconstructed args: -classpath -Yrepl-class-based -Yrepl-outdir /tmp/spark-83a0cda4-dd0b-472e-ad8b-a4b33b85f613/repl-06489815-5366-4aa0-9419-f01abda8d041
last tree to typer: TypeTree(class Byte)
tree position: line 6 of <console>
tree tpe: Byte
symbol: (final abstract) class Byte in package scala
symbol definition: final abstract class Byte extends (a ClassSymbol)
symbol package: scala
symbol owners: class Byte
call site: constructor $eval in object $eval in package $line21
== Source file context for tree position ==
3
4 object $eval {
5 lazy val $result = res0
6 lazy val $print: _root_.java.lang.String = {
7 $iw
8
9 "" )
+----------------------------+
|(INTERVAL '100' SECOND * c1)|
+----------------------------+
|null |
+----------------------------+
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org