You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Manoj Govindassamy (Jira)" <ji...@apache.org> on 2021/12/07 19:51:00 UTC
[jira] [Commented] (HUDI-2905) Insert crashes in MOR table with NullPointerException from HoodieMergeHandle
[ https://issues.apache.org/jira/browse/HUDI-2905?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17454820#comment-17454820 ]
Manoj Govindassamy commented on HUDI-2905:
------------------------------------------
This is related to https://issues.apache.org/jira/browse/HUDI-2894
The core here is the HFile reader is not reading the records reliably from the base file leading to read miss out and the follow on NPE when the returned list of partitions is empty. The issue was solved by https://github.com/apache/hudi/pull/4185
> Insert crashes in MOR table with NullPointerException from HoodieMergeHandle
> ----------------------------------------------------------------------------
>
> Key: HUDI-2905
> URL: https://issues.apache.org/jira/browse/HUDI-2905
> Project: Apache Hudi
> Issue Type: Task
> Reporter: Manoj Govindassamy
> Assignee: Manoj Govindassamy
> Priority: Blocker
> Fix For: 0.11.0
>
>
> Running Hoodie integration test suite with a MOR table type sometimes crashes with the following stack trace
> {noformat}
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
> at org.apache.spark.rdd.DoubleRDDFunctions.sum(DoubleRDDFunctions.scala:35)
> at org.apache.spark.api.java.JavaDoubleRDD.sum(JavaDoubleRDD.scala:165)
> at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:519)
> at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:306)
> at org.apache.hudi.integ.testsuite.HoodieDeltaStreamerWrapper.upsert(HoodieDeltaStreamerWrapper.java:44)
> at org.apache.hudi.integ.testsuite.HoodieDeltaStreamerWrapper.insert(HoodieDeltaStreamerWrapper.java:48)
> at org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter.insert(HoodieTestSuiteWriter.java:166)
> at org.apache.hudi.integ.testsuite.dag.nodes.InsertNode.ingest(InsertNode.java:70)
> at org.apache.hudi.integ.testsuite.dag.nodes.InsertNode.execute(InsertNode.java:53)
> at org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.executeNode(DagScheduler.java:139)
> ... 6 more
> Caused by: org.apache.hudi.exception.HoodieUpsertException: Error upserting bucketType UPDATE for partition :33
> at org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.handleUpsertPartition(BaseSparkCommitActionExecutor.java:320)
> at org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.handleInsertPartition(BaseSparkCommitActionExecutor.java:326)
> at org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.lambda$execute$ecf5068c$1(BaseSparkCommitActionExecutor.java:174)
> at org.apache.spark.api.java.JavaRDDLike.$anonfun$mapPartitionsWithIndex$1(JavaRDDLike.scala:102)
> at org.apache.spark.api.java.JavaRDDLike.$anonfun$mapPartitionsWithIndex$1$adapted(JavaRDDLike.scala:102)
> at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndex$2(RDD.scala:915)
> at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndex$2$adapted(RDD.scala:915)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386){noformat}
> Test properties
> {noformat}
> hoodie.insert.shuffle.parallelism=100
> hoodie.upsert.shuffle.parallelism=100
> hoodie.bulkinsert.shuffle.parallelism=100hoodie.deltastreamer.source.test.num_partitions=100
> hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
> hoodie.deltastreamer.source.test.max_unique_records=100000000
> hoodie.embed.timeline.server=false
> hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelectorhoodie.datasource.hive_sync.skip_ro_suffix=truehoodie.datasource.write.recordkey.field=_row_key
> hoodie.datasource.write.keygenerator.class=org.apache.hudi.keygen.TimestampBasedKeyGenerator
> hoodie.datasource.write.partitionpath.field=timestamphoodie.clustering.plan.strategy.sort.columns=_row_key
> hoodie.clustering.plan.strategy.daybased.lookback.partitions=0
> hoodie.clustering.inline.max.commits=1hoodie.deltastreamer.source.dfs.root=s3a://dl-scale-test/manoj/010RC2/integration-test-large-scale/slong/mor/input
> hoodie.deltastreamer.schemaprovider.target.schema.file=file:/home/hadoop/staging/source.avsc
> hoodie.deltastreamer.schemaprovider.source.schema.file=file:/home/hadoop/staging/source.avsc
> hoodie.deltastreamer.keygen.timebased.timestamp.type=UNIX_TIMESTAMP
> hoodie.deltastreamer.keygen.timebased.output.dateformat=yyyy/MM/ddhoodie.datasource.hive_sync.database=testdb{noformat}
> {noformat}
> /home/hadoop/spark-3.2.0-bin-hadoop3.2/bin/spark-submit \
> --packages org.apache.spark:spark-avro_2.12:3.2.0 \
> --conf spark.task.cpus=1 \
> --conf spark.executor.cores=1 \
> --conf spark.task.maxFailures=100 \
> --conf spark.memory.fraction=0.4 \
> --conf spark.rdd.compress=true \
> --conf spark.kryoserializer.buffer.max=2000m \
> --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
> --conf spark.memory.storageFraction=0.1 \
> --conf spark.shuffle.service.enabled=true \
> --conf spark.sql.hive.convertMetastoreParquet=false \
> --conf spark.driver.maxResultSize=12g \
> --conf spark.executor.heartbeatInterval=120s \
> --conf spark.network.timeout=600s \
> --conf spark.yarn.max.executor.failures=10 \
> --conf spark.sql.catalogImplementation=hive \
> --conf spark.hadoop.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain \
> --conf spark.hadoop.fs.s3a.connection.maximum=500 \
> --class org.apache.hudi.integ.testsuite.HoodieTestSuiteJob \
> /home/hadoop/staging/hudi-integ-test-bundle-0.10.0-rc2.jar \
> --source-ordering-field test_suite_source_ordering_field \
> --use-deltastreamer \
> --target-base-path s3a://dl-scale-test/manoj/010RC2/integration-test-large-scale/slong/mor/output \
> --input-base-path s3a://dl-scale-test/manoj/010RC2/integration-test-large-scale/slong/mor/input \
> --target-table table1 \
> --props file:/home/hadoop/staging/MERGE_ON_READ-true-test-slong.properties \
> --schemaprovider-class org.apache.hudi.integ.testsuite.schema.TestSuiteFileBasedSchemaProvider \
> --source-class org.apache.hudi.utilities.sources.AvroDFSSource \
> --input-file-size 125829120 \
> --workload-yaml-path file:/home/hadoop/staging/MERGE_ON_READ-true-slong.yaml \
> --workload-generator-classname org.apache.hudi.integ.testsuite.dag.WorkflowDagGenerator \
> --table-type MERGE_ON_READ \
> --compact-scheduling-minshare 1 \
> --clean-input \
> --clean-output{noformat}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)