You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "László Bodor (Jira)" <ji...@apache.org> on 2022/10/11 07:10:00 UTC
[jira] [Commented] (HIVE-26615) TestEncryptedHDFSCliDriver to use Tez
[ https://issues.apache.org/jira/browse/HIVE-26615?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17615531#comment-17615531 ]
László Bodor commented on HIVE-26615:
-------------------------------------
this is not as easy as I wished, a couple issues need to be addressed, e.g.
{code}
2022-10-11T00:01:40,127 ERROR [be3ae85f-c1a6-4c92-a694-2e73e4eb354c Listener at localhost/50845] FileOperations: Failed to read stats from filesystem
java.util.concurrent.ExecutionException: java.lang.IndexOutOfBoundsException: Index: 7735, Size: 0
at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:1.8.0_292]
at java.util.concurrent.FutureTask.get(FutureTask.java:192) ~[?:1.8.0_292]
at org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator.connect(FSStatsAggregator.java:94) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.stats.BasicStatsTask.createStatsAggregator(BasicStatsTask.java:441) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.stats.BasicStatsTask.aggregateStats(BasicStatsTask.java:282) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.stats.BasicStatsTask.process(BasicStatsTask.java:108) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:107) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:214) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:354) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:327) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:244) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:105) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:370) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:205) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:154) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:149) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:185) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:228) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:255) ~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processCmd1(CliDriver.java:200) ~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:126) ~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:421) ~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:352) ~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:727) ~[classes/:?]
at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:697) ~[classes/:?]
at org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:114) ~[classes/:?]
at org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:157) ~[classes/:?]
at org.apache.hadoop.hive.cli.TestEncryptedHDFSCliDriver.testCliDriver(TestEncryptedHDFSCliDriver.java:62) ~[test-classes/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_292]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_292]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_292]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_292]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.jar:4.13]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.jar:4.13]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.jar:4.13]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.jar:4.13]
at org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:135) ~[classes/:?]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.jar:4.13]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.jar:4.13]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.jar:4.13]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.jar:4.13]
at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.jar:4.13]
at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.jar:4.13]
at org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:95) ~[classes/:?]
at org.junit.rules.RunRules.evaluate(RunRules.java:20) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.jar:4.13]
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) ~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) ~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) ~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) ~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:377) ~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:138) ~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:465) ~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:451) ~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
Caused by: java.lang.IndexOutOfBoundsException: Index: 7735, Size: 0
at java.util.ArrayList.rangeCheck(ArrayList.java:659) ~[?:1.8.0_292]
at java.util.ArrayList.get(ArrayList.java:435) ~[?:1.8.0_292]
at org.apache.hive.com.esotericsoftware.kryo.util.MapReferenceResolver.getReadObject(MapReferenceResolver.java:83) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hive.com.esotericsoftware.kryo.Kryo.readReferenceOrNull(Kryo.java:855) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hive.com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:705) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readObject(SerializationUtilities.java:209) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator.lambda$connect$0(FSStatsAggregator.java:85) ~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_292]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_292]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_292]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_292]
{code}
> TestEncryptedHDFSCliDriver to use Tez
> -------------------------------------
>
> Key: HIVE-26615
> URL: https://issues.apache.org/jira/browse/HIVE-26615
> Project: Hive
> Issue Type: Improvement
> Reporter: László Bodor
> Assignee: László Bodor
> Priority: Major
> Labels: pull-request-available
> Attachments: org.apache.hadoop.hive.cli.TestEncryptedHDFSCliDriver-output.txt
>
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
> https://github.com/apache/hive/blob/eb860203fcdc126788e62e932b40a5f3ff944458/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java#L255-L261
> we set it to MR and then check if it's tez, which is already confusing
> {code}
> setClusterType(MiniClusterType.MR);
> setFsType(QTestMiniClusters.FsType.ENCRYPTED_HDFS); // override default FsType.HDFS
> if (getClusterType() == MiniClusterType.TEZ) {
> setHiveConfDir("data/conf/tez");
> } else {
> setHiveConfDir("data/conf");
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)