You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "sivabalan narayanan (Jira)" <ji...@apache.org> on 2022/04/25 15:52:00 UTC

[jira] [Updated] (HUDI-3972) MOR table read fails if no preCombine field is set

     [ https://issues.apache.org/jira/browse/HUDI-3972?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

sivabalan narayanan updated HUDI-3972:
--------------------------------------
    Priority: Blocker  (was: Major)

> MOR table read fails if no preCombine field is set
> --------------------------------------------------
>
>                 Key: HUDI-3972
>                 URL: https://issues.apache.org/jira/browse/HUDI-3972
>             Project: Apache Hudi
>          Issue Type: Bug
>          Components: spark
>            Reporter: sivabalan narayanan
>            Priority: Blocker
>             Fix For: 0.11.0
>
>
> When no preCombine field is set, hudi falls back to "ts" as default which goes into hoodie.properties. With MOR, while projecting only the required columns, we also project record key, and preCombine field and so "ts" is also being projected. and if original dataset does not have "ts", it fails. 
>  
> {code:java}
> java.util.NoSuchElementException: key not found: ts
> 	at scala.collection.MapLike.default(MapLike.scala:236)
> 	at scala.collection.MapLike.default$(MapLike.scala:235)
> 	at scala.collection.AbstractMap.default(Map.scala:65)
> 	at scala.collection.MapLike.apply(MapLike.scala:144)
> 	at scala.collection.MapLike.apply$(MapLike.scala:143)
> 	at scala.collection.AbstractMap.apply(Map.scala:65)
> 	at org.apache.hudi.HoodieSparkUtils$.$anonfun$getRequiredSchema$2(HoodieSparkUtils.scala:310)
> 	at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
> 	at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
> 	at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
> 	at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
> 	at scala.collection.TraversableLike.map(TraversableLike.scala:286)
> 	at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
> 	at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
> 	at org.apache.hudi.HoodieSparkUtils$.getRequiredSchema(HoodieSparkUtils.scala:310)
> 	at org.apache.hudi.HoodieBaseRelation.buildScan(HoodieBaseRelation.scala:173)
> 	at org.apache.spark.sql.execution.datasources.DataSourceStrategy$.$anonfun$apply$4(DataSourceStrategy.scala:323)
> 	at org.apache.spark.sql.execution.datasources.DataSourceStrategy$.$anonfun$pruneFilterProject$1(DataSourceStrategy.scala:357)
> 	at org.apache.spark.sql.execution.datasources.DataSourceStrategy$.pruneFilterProjectRaw(DataSourceStrategy.scala:413)
> 	at org.apache.spark.sql.execution.datasources.DataSourceStrategy$.pruneFilterProject(DataSourceStrategy.scala:356)
> 	at org.apache.spark.sql.execution.datasources.DataSourceStrategy$.apply(DataSourceStrategy.scala:323)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.$anonfun$plan$1(QueryPlanner.scala:63)
> 	at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
> 	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
> 	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:491)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.plan(QueryPlanner.scala:93)
> 	at org.apache.spark.sql.execution.SparkStrategies.plan(SparkStrategies.scala:68)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.$anonfun$plan$3(QueryPlanner.scala:78)
> 	at scala.collection.TraversableOnce$folder$1.apply(TraversableOnce.scala:196)
> 	at scala.collection.TraversableOnce$folder$1.apply(TraversableOnce.scala:194)
> 	at scala.collection.Iterator.foreach(Iterator.scala:943)
> 	at scala.collection.Iterator.foreach$(Iterator.scala:943)
> 	at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> 	at scala.collection.TraversableOnce.foldLeft(TraversableOnce.scala:199)
> 	at scala.collection.TraversableOnce.foldLeft$(TraversableOnce.scala:192)
> 	at scala.collection.AbstractIterator.foldLeft(Iterator.scala:1431)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.$anonfun$plan$2(QueryPlanner.scala:75)
> 	at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
> 	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.plan(QueryPlanner.scala:93)
> 	at org.apache.spark.sql.execution.SparkStrategies.plan(SparkStrategies.scala:68)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.$anonfun$plan$3(QueryPlanner.scala:78)
> 	at scala.collection.TraversableOnce$folder$1.apply(TraversableOnce.scala:196)
> 	at scala.collection.TraversableOnce$folder$1.apply(TraversableOnce.scala:194)
> 	at scala.collection.Iterator.foreach(Iterator.scala:943)
> 	at scala.collection.Iterator.foreach$(Iterator.scala:943)
> 	at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> 	at scala.collection.TraversableOnce.foldLeft(TraversableOnce.scala:199)
> 	at scala.collection.TraversableOnce.foldLeft$(TraversableOnce.scala:192)
> 	at scala.collection.AbstractIterator.foldLeft(Iterator.scala:1431)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.$anonfun$plan$2(QueryPlanner.scala:75)
> 	at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
> 	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
> 	at org.apache.spark.sql.catalyst.planning.QueryPlanner.plan(QueryPlanner.scala:93)
> 	at org.apache.spark.sql.execution.SparkStrategies.plan(SparkStrategies.scala:68)
> 	at org.apache.spark.sql.execution.QueryExecution$.createSparkPlan(QueryExecution.scala:468)
> 	at org.apache.spark.sql.execution.QueryExecution.$anonfun$sparkPlan$2(QueryExecution.scala:157)
> 	at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
> 	at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:196)
> 	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
> 	at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:196)
> 	at org.apache.spark.sql.execution.QueryExecution.$anonfun$sparkPlan$1(QueryExecution.scala:157)
> 	at org.apache.spark.sql.execution.QueryExecution.withCteMap(QueryExecution.scala:73)
> 	at org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:150)
> 	at org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:150)
> 	at org.apache.spark.sql.execution.QueryExecution.$anonfun$executedPlan$2(QueryExecution.scala:170)
> 	at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
> 	at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:196)
> 	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
> 	at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:196)
> 	at org.apache.spark.sql.execution.QueryExecution.$anonfun$executedPlan$1(QueryExecution.scala:170)
> 	at org.apache.spark.sql.execution.QueryExecution.withCteMap(QueryExecution.scala:73)
> 	at org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:163)
> 	at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:163)
> 	at org.apache.spark.sql.execution.QueryExecution.simpleString(QueryExecution.scala:214)
> 	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:259)
> 	at org.apache.spark.sql.execution.QueryExecution.explainString(QueryExecution.scala:228)
> 	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:98)
> 	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
> 	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
> 	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
> 	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
> 	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3704)
> 	at org.apache.spark.sql.Dataset.count(Dataset.scala:3011)
> 	at com.infinilake.sparkJobs.CompareSchemas.getHoodieTableRowCount(CompareSchemas.java:128)
> 	at com.infinilake.sparkJobs.CompareSchemas.getCounts(CompareSchemas.java:133)
> 	at com.infinilake.sparkJobs.CompareSchemas.lambda$getSchemaAndCounts$8(CompareSchemas.java:198)
> 	at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> 	at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:647)
> 	at com.infinilake.sparkJobs.CompareSchemas.getSchemaAndCounts(CompareSchemas.java:166)
> 	at com.infinilake.sparkJobs.CompareSchemas.main(CompareSchemas.java:258)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> 	at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
> 	at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
> 	at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
> 	at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
> 	at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
> 	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
> 	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) {code}



--
This message was sent by Atlassian Jira
(v8.20.7#820007)