[jira] [Created] (HUDI-3334) Unable to merge HoodieMetadataPayload during partition listing

["Ethan Guo (Jira)" <ji...@apache.org>]

Ethan Guo created HUDI-3334:
-------------------------------

             Summary: Unable to merge HoodieMetadataPayload during partition listing
                 Key: HUDI-3334
                 URL: https://issues.apache.org/jira/browse/HUDI-3334
             Project: Apache Hudi
          Issue Type: Task
            Reporter: Ethan Guo
             Fix For: 0.11.0


Stacktrace:
{code:java}
Caused by: org.apache.hudi.exception.HoodieException: Error fetching partition paths from metadata table	at org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:299)	at org.apache.hudi.HoodieTableFileIndexBase.getAllQueryPartitionPaths(HoodieTableFileIndexBase.scala:233)	at org.apache.hudi.HoodieTableFileIndexBase.loadPartitionPathFiles(HoodieTableFileIndexBase.scala:195)	at org.apache.hudi.HoodieTableFileIndexBase.refresh0(HoodieTableFileIndexBase.scala:108)	at org.apache.hudi.HoodieTableFileIndexBase.<init>(HoodieTableFileIndexBase.scala:88)	at org.apache.hudi.hadoop.HiveHoodieTableFileIndex.<init>(HiveHoodieTableFileIndex.java:52)	at org.apache.hudi.hadoop.HoodieFileInputFormatBase.listStatusForSnapshotMode(HoodieFileInputFormatBase.java:170)	at org.apache.hudi.hadoop.HoodieFileInputFormatBase.listStatus(HoodieFileInputFormatBase.java:141)	at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:322)	at org.apache.hadoop.hive.ql.io.HiveInputFormat.addSplitsForGroup(HiveInputFormat.java:442)	at org.apache.hadoop.hive.ql.io.HiveInputFormat.getSplits(HiveInputFormat.java:561)	at org.apache.hadoop.mapreduce.JobSubmitter.writeOldSplits(JobSubmitter.java:330)	at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:322)	at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:198)	at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1341)	at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1338)	at java.security.AccessController.doPrivileged(Native Method)	at javax.security.auth.Subject.doAs(Subject.java:422)	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840)	at org.apache.hadoop.mapreduce.Job.submit(Job.java:1338)	at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:575)	at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:570)	at java.security.AccessController.doPrivileged(Native Method)	at javax.security.auth.Subject.doAs(Subject.java:422)	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840)	at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:570)	at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:561)	at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:411)	at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:151)	at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:199)	at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:100)	at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2183)	at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1839)	at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1526)	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1237)	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1232)	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:255)	... 11 moreCaused by: org.apache.hudi.exception.HoodieMetadataException: Failed to retrieve list of partition from metadata	at org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:100)	at org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:297)	... 47 moreCaused by: org.apache.hudi.exception.HoodieException: Exception when reading log file 	at org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:333)	at org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:179)	at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.performScan(HoodieMergedLogRecordScanner.java:103)	at org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.<init>(HoodieMetadataMergedLogRecordReader.java:71)	at org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.<init>(HoodieMetadataMergedLogRecordReader.java:51)	at org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader$Builder.build(HoodieMetadataMergedLogRecordReader.java:246)	at org.apache.hudi.metadata.HoodieBackedTableMetadata.getLogRecordScanner(HoodieBackedTableMetadata.java:346)	at org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$openReadersIfNeeded$2(HoodieBackedTableMetadata.java:262)	at java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)	at org.apache.hudi.metadata.HoodieBackedTableMetadata.openReadersIfNeeded(HoodieBackedTableMetadata.java:239)	at org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:129)	at org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:124)	at org.apache.hudi.metadata.BaseTableMetadata.fetchAllPartitionPaths(BaseTableMetadata.java:154)	at org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:98)	... 48 moreCaused by: java.lang.IllegalArgumentException: Cannot combine 2 with 1	at org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)	at org.apache.hudi.metadata.HoodieMetadataPayload.preCombine(HoodieMetadataPayload.java:141)	at org.apache.hudi.metadata.HoodieMetadataPayload.preCombine(HoodieMetadataPayload.java:63)	at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.processNextRecord(HoodieMergedLogRecordScanner.java:144)	at org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.processNextRecord(HoodieMetadataMergedLogRecordReader.java:78)	at org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.processDataBlock(AbstractHoodieLogRecordReader.java:369)	at org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.processQueuedBlocksForInstant(AbstractHoodieLogRecordReader.java:431)	at org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:239)	... 61 more
	at org.apache.hive.jdbc.HiveStatement.waitForOperationToComplete(HiveStatement.java:385)	at org.apache.hive.jdbc.HiveStatement.execute(HiveStatement.java:254)	at org.apache.hive.jdbc.HiveStatement.executeQuery(HiveStatement.java:476)	at org.apache.hudi.cli.utils.HiveUtil.countRecords(HiveUtil.java:58)	at org.apache.hudi.cli.commands.HoodieSyncCommand.validateSync(HoodieSyncCommand.java:69)	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)	at java.lang.reflect.Method.invoke(Method.java:498)	at org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:216) {code}
https://gist.github.com/yihua/2c7481fbbd509ca7a81e5d0ea7efc2fd



--
This message was sent by Atlassian Jira
(v8.20.1#820001)