You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by GitBox <gi...@apache.org> on 2021/11/02 23:18:26 UTC
[GitHub] [hudi] nsivabalan commented on a change in pull request #3880: [HUDI-2606][WIP] Enable metadata reader by default
nsivabalan commented on a change in pull request #3880:
URL: https://github.com/apache/hudi/pull/3880#discussion_r741526371
##########
File path: packaging/hudi-hive-sync-bundle/pom.xml
##########
@@ -69,12 +69,18 @@
<include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-hive-sync</include>
+ <include>org.apache.hbase:hbase-client</include>
Review comment:
Integ test failure stacktrace
```
2021-10-27T17:37:02.0772040Z #################################################################################################
2021-10-27T17:37:02.0774104Z 76786 [main] INFO org.apache.hudi.integ.ITTestBase - Container : /adhoc-1, Running command :spark-submit --class org.apache.hudi.hive.HiveSyncTool /var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-sync-bundle.jar --database default --table stock_ticks_cow --base-path /user/hive/warehouse/stock_ticks_cow --base-file-format PARQUET --user hive --pass hive --jdbc-url jdbc:hive2://hiveserver:10000 --partitioned-by dt
2021-10-27T17:37:02.0775664Z 76786 [main] INFO org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:02.0776313Z #################################################################################################
2021-10-27T17:37:08.2649560Z 82970 [dockerjava-jaxrs-async-6] INFO org.apache.hudi.integ.ITTestBase - onComplete called
2021-10-27T17:37:08.2650772Z 82973 [main] INFO org.apache.hudi.integ.ITTestBase - Exit code for command : 1
2021-10-27T17:37:08.2651626Z 82973 [main] ERROR org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2651987Z
2021-10-27T17:37:08.2652302Z ###### Stdout #######
2021-10-27T17:37:08.2652511Z
2021-10-27T17:37:08.2653143Z 82973 [main] ERROR org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2653492Z
2021-10-27T17:37:08.2653794Z ###### Stderr #######
2021-10-27T17:37:08.2654735Z 21/10/27 17:37:03 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2021-10-27T17:37:08.2655593Z Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
2021-10-27T17:37:08.2656511Z at org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex.<clinit>(HFileBootstrapIndex.java:92)
2021-10-27T17:37:08.2657147Z at java.lang.Class.forName0(Native Method)
2021-10-27T17:37:08.2657684Z at java.lang.Class.forName(Class.java:264)
2021-10-27T17:37:08.2658313Z at org.apache.hudi.common.util.ReflectionUtils.getClass(ReflectionUtils.java:54)
2021-10-27T17:37:08.2659027Z at org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:89)
2021-10-27T17:37:08.2659750Z at org.apache.hudi.common.bootstrap.index.BootstrapIndex.getBootstrapIndex(BootstrapIndex.java:163)
2021-10-27T17:37:08.2660565Z at org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:108)
2021-10-27T17:37:08.2661393Z at org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:106)
2021-10-27T17:37:08.2662210Z at org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:100)
2021-10-27T17:37:08.2662997Z at org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:91)
2021-10-27T17:37:08.2663883Z at org.apache.hudi.metadata.HoodieTableMetadataUtil.loadPartitionFileGroupsWithLatestFileSlices(HoodieTableMetadataUtil.java:352)
2021-10-27T17:37:08.2664793Z at org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$openReadersIfNeeded$2(HoodieBackedTableMetadata.java:185)
2021-10-27T17:37:08.2665914Z at java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
2021-10-27T17:37:08.2666733Z at org.apache.hudi.metadata.HoodieBackedTableMetadata.openReadersIfNeeded(HoodieBackedTableMetadata.java:176)
2021-10-27T17:37:08.2667568Z at org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKeyFromMetadata(HoodieBackedTableMetadata.java:124)
2021-10-27T17:37:08.2668390Z at org.apache.hudi.metadata.BaseTableMetadata.fetchAllPartitionPaths(BaseTableMetadata.java:153)
2021-10-27T17:37:08.2669162Z at org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:94)
2021-10-27T17:37:08.2669904Z at org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:281)
2021-10-27T17:37:08.2670666Z at org.apache.hudi.sync.common.AbstractSyncHoodieClient.getPartitionsWrittenToSince(AbstractSyncHoodieClient.java:157)
2021-10-27T17:37:08.2671447Z at org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:191)
2021-10-27T17:37:08.2672111Z at org.apache.hudi.hive.HiveSyncTool.doSync(HiveSyncTool.java:131)
2021-10-27T17:37:08.2672783Z at org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:117)
2021-10-27T17:37:08.2673420Z at org.apache.hudi.hive.HiveSyncTool.main(HiveSyncTool.java:359)
2021-10-27T17:37:08.2679252Z at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2021-10-27T17:37:08.2679805Z at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2021-10-27T17:37:08.2680439Z at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2021-10-27T17:37:08.2680986Z at java.lang.reflect.Method.invoke(Method.java:498)
2021-10-27T17:37:08.2681730Z at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
2021-10-27T17:37:08.2682398Z at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
2021-10-27T17:37:08.2683025Z at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
2021-10-27T17:37:08.2683600Z at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
2021-10-27T17:37:08.2684159Z at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
2021-10-27T17:37:08.2684738Z at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
2021-10-27T17:37:08.2685438Z at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
2021-10-27T17:37:08.2685953Z at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2021-10-27T17:37:08.2686470Z Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.util.Bytes
2021-10-27T17:37:08.2687071Z at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
2021-10-27T17:37:08.2687764Z at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
2021-10-27T17:37:08.2688262Z at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
2021-10-27T17:37:08.2688635Z ... 35 more
2021-10-27T17:37:08.2688791Z
2021-10-27T17:37:08.2689569Z 82974 [main] WARN org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2689775Z
2021-10-27T17:37:08.2691310Z :::::: Original Command : [spark-submit, --class, org.apache.hudi.hive.HiveSyncTool, /var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-sync-bundle.jar, --database, default, --table, stock_ticks_cow, --base-path, /user/hive/warehouse/stock_ticks_cow, --base-file-format, PARQUET, --user, hive, --pass, hive, --jdbc-url, jdbc:hive2://hiveserver:10000, --partitioned-by, dt]
2021-10-27T17:37:08.2692540Z 82974 [main] WARN org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2692747Z
2021-10-27T17:37:08.2693272Z :::::: Stderr of timed-out command ::::::
2021-10-27T17:37:08.2694154Z 21/10/27 17:37:03 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2021-10-27T17:37:08.2694870Z Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
2021-10-27T17:37:08.2695520Z at org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex.<clinit>(HFileBootstrapIndex.java:92)
2021-10-27T17:37:08.2696177Z at java.lang.Class.forName0(Native Method)
2021-10-27T17:37:08.2696599Z at java.lang.Class.forName(Class.java:264)
2021-10-27T17:37:08.2697097Z at org.apache.hudi.common.util.ReflectionUtils.getClass(ReflectionUtils.java:54)
2021-10-27T17:37:08.2697706Z at org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:89)
2021-10-27T17:37:08.2698350Z at org.apache.hudi.common.bootstrap.index.BootstrapIndex.getBootstrapIndex(BootstrapIndex.java:163)
2021-10-27T17:37:08.2699033Z at org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:108)
2021-10-27T17:37:08.2699759Z at org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:106)
2021-10-27T17:37:08.2700537Z at org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:100)
2021-10-27T17:37:08.2701199Z at org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:91)
2021-10-27T17:37:08.2701889Z at org.apache.hudi.metadata.HoodieTableMetadataUtil.loadPartitionFileGroupsWithLatestFileSlices(HoodieTableMetadataUtil.java:352)
2021-10-27T17:37:08.2702634Z at org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$openReadersIfNeeded$2(HoodieBackedTableMetadata.java:185)
2021-10-27T17:37:08.2703280Z at java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
2021-10-27T17:37:08.2704079Z at org.apache.hudi.metadata.HoodieBackedTableMetadata.openReadersIfNeeded(HoodieBackedTableMetadata.java:176)
2021-10-27T17:37:08.2704827Z at org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKeyFromMetadata(HoodieBackedTableMetadata.java:124)
2021-10-27T17:37:08.2705610Z at org.apache.hudi.metadata.BaseTableMetadata.fetchAllPartitionPaths(BaseTableMetadata.java:153)
2021-10-27T17:37:08.2706276Z at org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:94)
2021-10-27T17:37:08.2707029Z at org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:281)
2021-10-27T17:37:08.2707709Z at org.apache.hudi.sync.common.AbstractSyncHoodieClient.getPartitionsWrittenToSince(AbstractSyncHoodieClient.java:157)
2021-10-27T17:37:08.2708388Z at org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:191)
2021-10-27T17:37:08.2708930Z at org.apache.hudi.hive.HiveSyncTool.doSync(HiveSyncTool.java:131)
2021-10-27T17:37:08.2709491Z at org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:117)
2021-10-27T17:37:08.2710045Z at org.apache.hudi.hive.HiveSyncTool.main(HiveSyncTool.java:359)
2021-10-27T17:37:08.2710527Z at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2021-10-27T17:37:08.2711056Z at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2021-10-27T17:37:08.2711675Z at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2021-10-27T17:37:08.2712214Z at java.lang.reflect.Method.invoke(Method.java:498)
2021-10-27T17:37:08.2712756Z at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
2021-10-27T17:37:08.2713418Z at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
2021-10-27T17:37:08.2714150Z at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
2021-10-27T17:37:08.2714662Z at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
2021-10-27T17:37:08.2715187Z at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
2021-10-27T17:37:08.2715751Z at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
2021-10-27T17:37:08.2716310Z at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
2021-10-27T17:37:08.2716807Z at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2021-10-27T17:37:08.2717318Z Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.util.Bytes
2021-10-27T17:37:08.2717836Z at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
2021-10-27T17:37:08.2718421Z at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
2021-10-27T17:37:08.2718923Z at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
2021-10-27T17:37:08.2719294Z ... 35 more
2021-10-27T17:37:08.2719474Z
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org