You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bigtop.apache.org by "Kengo Seki (Jira)" <ji...@apache.org> on 2020/11/02 01:12:00 UTC
[jira] [Commented] (BIGTOP-3429) QFS smoke test fails on Ubuntu
16.04
[ https://issues.apache.org/jira/browse/BIGTOP-3429?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17224368#comment-17224368 ]
Kengo Seki commented on BIGTOP-3429:
------------------------------------
I could reproduce this issue on Ubuntu 16.04 locally. According to the NodeManager's log, it looks like failing to load QFS' native library. But I'm not sure why it occurs only on Ubuntu 16.04 for now.
{code:title=/var/log/hadoop-yarn/yarn-yarn-nodemanager-*.log}
2020-11-02 00:34:26,790 WARN org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService: { qfs://20160a27bb51.bigtop.apache.org:20000/user/yarn/.staging/job_1604276770710_0001/job.splitmetainfo, 1604277259223, FILE, null } failed:
java.lang.ExceptionInInitializerError
at com.quantcast.qfs.hadoop.QFSImpl.<init>(QFSImpl.java:46)
at com.quantcast.qfs.hadoop.QuantcastFileSystem.createIFSImpl(QuantcastFileSystem.java:95)
at com.quantcast.qfs.hadoop.QuantcastFileSystem.initialize(QuantcastFileSystem.java:76)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3242)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:121)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3291)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3259)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:470)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
at org.apache.hadoop.yarn.util.FSDownload.copy(FSDownload.java:251)
at org.apache.hadoop.yarn.util.FSDownload.access$000(FSDownload.java:63)
at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:366)
at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:364)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893)
at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:364)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.doDownloadCall(ContainerLocalizer.java:241)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.call(ContainerLocalizer.java:234)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.call(ContainerLocalizer.java:222)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Unable to load qfs_access native library
at com.quantcast.qfs.access.KfsAccess.<clinit>(KfsAccess.java:236)
... 26 more
Caused by: java.lang.UnsatisfiedLinkError: no qfs_access in java.library.path
at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1860)
at java.lang.Runtime.loadLibrary0(Runtime.java:871)
at java.lang.System.loadLibrary(System.java:1124)
at com.quantcast.qfs.access.KfsAccess.<clinit>(KfsAccess.java:234)
... 26 more
Caused by: Unable to load qfs_access native library
java.lang.RuntimeException: Unable to load qfs_access native library
at com.quantcast.qfs.access.KfsAccess.<clinit>(KfsAccess.java:236)
at com.quantcast.qfs.hadoop.QFSImpl.<init>(QFSImpl.java:46)
at com.quantcast.qfs.hadoop.QuantcastFileSystem.createIFSImpl(QuantcastFileSystem.java:95)
at com.quantcast.qfs.hadoop.QuantcastFileSystem.initialize(QuantcastFileSystem.java:76)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3242)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:121)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3291)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3259)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:470)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
at org.apache.hadoop.yarn.util.FSDownload.copy(FSDownload.java:251)
at org.apache.hadoop.yarn.util.FSDownload.access$000(FSDownload.java:63)
at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:366)
at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:364)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893)
at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:364)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.doDownloadCall(ContainerLocalizer.java:241)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.call(ContainerLocalizer.java:234)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.call(ContainerLocalizer.java:222)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.UnsatisfiedLinkError: no qfs_access in java.library.path
at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1860)
at java.lang.Runtime.loadLibrary0(Runtime.java:871)
at java.lang.System.loadLibrary(System.java:1124)
at com.quantcast.qfs.access.KfsAccess.<clinit>(KfsAccess.java:234)
... 26 more
Caused by: no qfs_access in java.library.path
java.lang.UnsatisfiedLinkError: no qfs_access in java.library.path
at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1860)
at java.lang.Runtime.loadLibrary0(Runtime.java:871)
at java.lang.System.loadLibrary(System.java:1124)
at com.quantcast.qfs.access.KfsAccess.<clinit>(KfsAccess.java:234)
at com.quantcast.qfs.hadoop.QFSImpl.<init>(QFSImpl.java:46)
at com.quantcast.qfs.hadoop.QuantcastFileSystem.createIFSImpl(QuantcastFileSystem.java:95)
at com.quantcast.qfs.hadoop.QuantcastFileSystem.initialize(QuantcastFileSystem.java:76)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3242)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:121)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3291)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3259)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:470)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
at org.apache.hadoop.yarn.util.FSDownload.copy(FSDownload.java:251)
at org.apache.hadoop.yarn.util.FSDownload.access$000(FSDownload.java:63)
at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:366)
at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:364)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1893)
at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:364)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.doDownloadCall(ContainerLocalizer.java:241)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.call(ContainerLocalizer.java:234)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer$FSDownloadWrapper.call(ContainerLocalizer.java:222)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
{code}
> QFS smoke test fails on Ubuntu 16.04
> ------------------------------------
>
> Key: BIGTOP-3429
> URL: https://issues.apache.org/jira/browse/BIGTOP-3429
> Project: Bigtop
> Issue Type: Bug
> Components: qfs, tests
> Reporter: Kengo Seki
> Priority: Major
>
> Even after merging BIGTOP-3413, QFS' smoke test is failing on this distro for some reason.
> https://ci.bigtop.apache.org/job/Bigtop-trunk-smoke-tests/181/COMPONENTS=hdfs.yarn.qfs@qfs,OS=ubuntu-16.04-amd64-deploy/console
> https://ci.bigtop.apache.org/job/Bigtop-trunk-smoke-tests/181/COMPONENTS=hdfs.yarn.qfs@qfs,OS=ubuntu-16.04-arm64-deploy/console
> {code}
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples STANDARD_ERROR
> 20/09/30 04:26:01 INFO lang.Object: Failed to unpack jar resources. Attemting to use bigtop sources
> 20/09/30 04:26:01 INFO lang.Object: MAKING DIRECTORIES ..................... examples examples-output
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[pi] FAILED
> java.lang.AssertionError: Example pi /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 2 1000 failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCallStatic(CallSiteArray.java:55)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:196)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[wordcount] FAILED
> java.lang.AssertionError: Example wordcount /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar wordcount examples/text examples-output/wordcount failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[multifilewc] FAILED
> java.lang.AssertionError: Example multifilewc /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar multifilewc examples/text examples-output/multifilewc failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[aggregatewordcount] FAILED
> java.lang.AssertionError: Example aggregatewordcount /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar aggregatewordcount examples/text examples-output/aggregatewordcount 2 textinputformat failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[aggregatewordhist] FAILED
> java.lang.AssertionError: Example aggregatewordhist /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar aggregatewordhist examples/text examples-output/aggregatewordhist 2 textinputformat failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[grep] FAILED
> java.lang.AssertionError: Example grep /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar grep examples/text examples-output/grep '[Cc]uriouser' failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[secondarysort] FAILED
> java.lang.AssertionError: Example secondarysort /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar secondarysort examples/ints examples-output/secondarysort failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples > testMRExample[randomtextwriter] FAILED
> java.lang.AssertionError: Example randomtextwriter /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar randomtextwriter -D mapreduce.randomtextwriter.totalbytes=1073741824 examples-output/randomtextwriter failed
> at org.junit.Assert.fail(Assert.java:88)
> at org.junit.Assert.assertTrue(Assert.java:41)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:104)
> at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:326)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite$StaticMetaMethodSiteNoUnwrap.invoke(StaticMetaMethodSite.java:133)
> at org.codehaus.groovy.runtime.callsite.StaticMetaMethodSite.callStatic(StaticMetaMethodSite.java:102)
> at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callStatic(AbstractCallSite.java:216)
> at org.apache.bigtop.itest.hadoop.mapreduce.TestHadoopExamples.testMRExample(TestHadoopExamples.groovy:176)
> Gradle Test Executor 2 finished executing tests.
> > Task :bigtop-tests:smoke-tests:qfs:test
> 8 tests completed, 8 failed
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)