You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@accumulo.apache.org by "Webb, Ryan L." <Ry...@jhuapl.edu> on 2015/07/28 16:34:53 UTC

GC and Monitor Unable to scan login as !SYSTEM

Below are two stack traces - one is from the GC and the latter is form the monitor when I try to look at a specific table.
I am using 1.6.0-CHD5.1.0 on CentOS 6.4

This happened after I attempted to fix the FQDN issue with bulk table loading.

Is there a way to reset the password? How did this get changed?


Unable to scan metadata table
                java.lang.RuntimeException: org.apache.accumulo.core.client.AccumuloSecurityException: Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.root(ID:+r) - Username or Password is Invalid
                                at org.apache.accumulo.core.client.impl.ScannerIterator.hasNext(ScannerIterator.java:187)
                                at org.apache.accumulo.server.util.MetadataTableUtil.getLogEntries(MetadataTableUtil.java:547)
                                at org.apache.accumulo.server.util.MetadataTableUtil$LogEntryIterator.<init>(MetadataTableUtil.java:615)
                                at org.apache.accumulo.server.util.MetadataTableUtil.getLogEntries(MetadataTableUtil.java:652)
                                at org.apache.accumulo.gc.GarbageCollectWriteAheadLogs.removeMetadataEntries(GarbageCollectWriteAheadLogs.java:280)
                                at org.apache.accumulo.gc.GarbageCollectWriteAheadLogs.collect(GarbageCollectWriteAheadLogs.java:125)
                                at org.apache.accumulo.gc.SimpleGarbageCollector.run(SimpleGarbageCollector.java:568)
                                at org.apache.accumulo.gc.SimpleGarbageCollector.main(SimpleGarbageCollector.java:154)
                                at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
                                at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
                                at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                                at java.lang.reflect.Method.invoke(Method.java:606)
                                at org.apache.accumulo.start.Main$1.run(Main.java:141)
                                at java.lang.Thread.run(Thread.java:744)
                Caused by: org.apache.accumulo.core.client.AccumuloSecurityException: Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.root(ID:+r) - Username or Password is Invalid
                                at org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.java:454)
                                at org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.java:277)
                                at org.apache.accumulo.core.client.impl.ScannerIterator$Reader.run(ScannerIterator.java:84)
                                at org.apache.accumulo.core.client.impl.ScannerIterator.hasNext(ScannerIterator.java:177)
                                ... 13 more
                Caused by: ThriftSecurityException(user:!SYSTEM, code:BAD_CREDENTIALS)
                                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startScan_result$startScan_resultStandardScheme.read(TabletClientService.java:5100)
                                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startScan_result$startScan_resultStandardScheme.read(TabletClientService.java:5077)
                                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startScan_result.read(TabletClientService.java:5000)
                                at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)
                                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.recv_startScan(TabletClientService.java:226)
                                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.startScan(TabletClientService.java:202)
                                at org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.java:399)
                                ... 16 more


java.lang.RuntimeException: org.apache.accumulo.core.client.AccumuloSecurityException: Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.metadata(ID:!0) - Username or Password is Invalid
                at org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.hasNext(TabletServerBatchReaderIterator.java:208)
                at org.apache.accumulo.server.master.state.MetaDataTableScanner.hasNext(MetaDataTableScanner.java:117)
                at org.apache.accumulo.monitor.servlets.TablesServlet.doTableDetails(TablesServlet.java:158)
                at org.apache.accumulo.monitor.servlets.TablesServlet.pageBody(TablesServlet.java:75)
                at org.apache.accumulo.monitor.servlets.BasicServlet.doGet(BasicServlet.java:65)
                at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
                at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
                at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
                at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:401)
                at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
                at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
                at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
                at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
                at org.mortbay.jetty.Server.handle(Server.java:326)
                at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
                at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
                at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
                at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
                at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
                at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
                at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: org.apache.accumulo.core.client.AccumuloSecurityException: Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.metadata(ID:!0) - Username or Password is Invalid
                at org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.doLookup(TabletServerBatchReaderIterator.java:716)
                at org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator$QueryTask.run(TabletServerBatchReaderIterator.java:372)
                at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
                at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
                at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
                at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
                at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
                at java.lang.Thread.run(Thread.java:744)
Caused by: ThriftSecurityException(user:!SYSTEM, code:BAD_CREDENTIALS)
                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startMultiScan_result$startMultiScan_resultStandardScheme.read(TabletClientService.java:8613)
                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startMultiScan_result$startMultiScan_resultStandardScheme.read(TabletClientService.java:8590)
                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startMultiScan_result.read(TabletClientService.java:8529)
                at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)
                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.recv_startMultiScan(TabletClientService.java:311)
                at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.startMultiScan(TabletClientService.java:291)
                at org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.doLookup(TabletServerBatchReaderIterator.java:658)
                ... 7 more

RE: GC and Monitor Unable to scan login as !SYSTEM

Posted by "Webb, Ryan L." <Ry...@jhuapl.edu>.
Alright, Thanks for the information. 
I was updating instance.volumes to use FQDN and didn't add it to all of the roles -- Providing it to all solved the issue.

Thanks for the help!
Ryan Webb

-----Original Message-----
From: Josh Elser [mailto:josh.elser@gmail.com] 
Sent: Tuesday, July 28, 2015 12:27 PM
To: user@accumulo.apache.org
Subject: Re: GC and Monitor Unable to scan login as !SYSTEM

Accumulo has an internal user which it uses for RPCs between Accumulo serverse. The "!SYSTEM" user is this user. This user also has credentials but it is computed from a number of things, one of which is the value of all properties beginning with "instance." in accumulo-site.xml.

If you are seeing errors, it is likely that you did not correctly update accumulo-site.xml on _all_ nodes in the system and, as such, some nodes are expecting one "password" for !SYSTEM with other nodes expecting a different one. Ensure that you have consistent configuration across all nodes and restart Accumulo.

I think you have fooled yourself that the second exception is due to you reading from a user table. Note again, the user in question is the !SYSTEM user. You are not this user.

Webb, Ryan L. wrote:
> Below are two stack traces - one is from the GC and the latter is form 
> the monitor when I try to look at a specific table.
>
> I am using 1.6.0-CHD5.1.0 on CentOS 6.4
>
> This happened after I attempted to fix the FQDN issue with bulk table 
> loading.
>
> Is there a way to reset the password? How did this get changed?
>
> Unable to scan metadata table
>
> java.lang.RuntimeException:
> org.apache.accumulo.core.client.AccumuloSecurityException: Error 
> BAD_CREDENTIALS for user !SYSTEM on table accumulo.root(ID:+r) - 
> Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.ScannerIterator.hasNext(ScannerIt
> erator.java:187)
>
> at
> org.apache.accumulo.server.util.MetadataTableUtil.getLogEntries(Metada
> taTableUtil.java:547)
>
> at
> org.apache.accumulo.server.util.MetadataTableUtil$LogEntryIterator.<in
> it>(MetadataTableUtil.java:615)
>
> at
> org.apache.accumulo.server.util.MetadataTableUtil.getLogEntries(Metada
> taTableUtil.java:652)
>
> at
> org.apache.accumulo.gc.GarbageCollectWriteAheadLogs.removeMetadataEntr
> ies(GarbageCollectWriteAheadLogs.java:280)
>
> at
> org.apache.accumulo.gc.GarbageCollectWriteAheadLogs.collect(GarbageCol
> lectWriteAheadLogs.java:125)
>
> at
> org.apache.accumulo.gc.SimpleGarbageCollector.run(SimpleGarbageCollect
> or.java:568)
>
> at
> org.apache.accumulo.gc.SimpleGarbageCollector.main(SimpleGarbageCollec
> tor.java:154)
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.j
> ava:57)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccess
> orImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:606)
>
> at org.apache.accumulo.start.Main$1.run(Main.java:141)
>
> at java.lang.Thread.run(Thread.java:744)
>
> Caused by: org.apache.accumulo.core.client.AccumuloSecurityException:
> Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.root(ID:+r) - 
> Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.
> java:454)
>
> at
> org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.
> java:277)
>
> at
> org.apache.accumulo.core.client.impl.ScannerIterator$Reader.run(Scanne
> rIterator.java:84)
>
> at
> org.apache.accumulo.core.client.impl.ScannerIterator.hasNext(ScannerIt
> erator.java:177)
>
> ... 13 more
>
> Caused by: ThriftSecurityException(user:!SYSTEM, code:BAD_CREDENTIALS)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$start
> Scan_result$startScan_resultStandardScheme.read(TabletClientService.ja
> va:5100)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$start
> Scan_result$startScan_resultStandardScheme.read(TabletClientService.ja
> va:5077)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$start
> Scan_result.read(TabletClientService.java:5000)
>
> at 
> org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Clien
> t.recv_startScan(TabletClientService.java:226)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Clien
> t.startScan(TabletClientService.java:202)
>
> at
> org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.
> java:399)
>
> ... 16 more
>
> java.lang.RuntimeException:
> org.apache.accumulo.core.client.AccumuloSecurityException: Error 
> BAD_CREDENTIALS for user !SYSTEM on table accumulo.metadata(ID:!0) - 
> Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.h
> asNext(TabletServerBatchReaderIterator.java:208)
>
> at
> org.apache.accumulo.server.master.state.MetaDataTableScanner.hasNext(M
> etaDataTableScanner.java:117)
>
> at
> org.apache.accumulo.monitor.servlets.TablesServlet.doTableDetails(Tabl
> esServlet.java:158)
>
> at
> org.apache.accumulo.monitor.servlets.TablesServlet.pageBody(TablesServ
> let.java:75)
>
> at
> org.apache.accumulo.monitor.servlets.BasicServlet.doGet(BasicServlet.j
> ava:65)
>
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>
> at 
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>
> at 
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:40
> 1)
>
> at 
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:18
> 2)
>
> at 
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:76
> 6)
>
> at
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandl
> erCollection.java:230)
>
> at 
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:15
> 2)
>
> at org.mortbay.jetty.Server.handle(Server.java:326)
>
> at 
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542
> )
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpCon
> nection.java:928)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>
> at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>
> at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>
> at
> org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.j
> ava:228)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.ja
> va:582)
>
> Caused by: org.apache.accumulo.core.client.AccumuloSecurityException:
> Error BAD_CREDENTIALS for user !SYSTEM on table 
> accumulo.metadata(ID:!0)
> - Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.d
> oLookup(TabletServerBatchReaderIterator.java:716)
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator$Q
> ueryTask.run(TabletServerBatchReaderIterator.java:372)
>
> at
> org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.j
> ava:47)
>
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.j
> ava:1145)
>
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.
> java:615)
>
> at
> org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.j
> ava:47)
>
> at
> org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java
> :34)
>
> at java.lang.Thread.run(Thread.java:744)
>
> Caused by: ThriftSecurityException(user:!SYSTEM, code:BAD_CREDENTIALS)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$start
> MultiScan_result$startMultiScan_resultStandardScheme.read(TabletClient
> Service.java:8613)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$start
> MultiScan_result$startMultiScan_resultStandardScheme.read(TabletClient
> Service.java:8590)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$start
> MultiScan_result.read(TabletClientService.java:8529)
>
> at 
> org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Clien
> t.recv_startMultiScan(TabletClientService.java:311)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Clien
> t.startMultiScan(TabletClientService.java:291)
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.d
> oLookup(TabletServerBatchReaderIterator.java:658)
>
> ... 7 more
>

Re: GC and Monitor Unable to scan login as !SYSTEM

Posted by Josh Elser <jo...@gmail.com>.
Accumulo has an internal user which it uses for RPCs between Accumulo 
serverse. The "!SYSTEM" user is this user. This user also has 
credentials but it is computed from a number of things, one of which is 
the value of all properties beginning with "instance." in accumulo-site.xml.

If you are seeing errors, it is likely that you did not correctly update 
accumulo-site.xml on _all_ nodes in the system and, as such, some nodes 
are expecting one "password" for !SYSTEM with other nodes expecting a 
different one. Ensure that you have consistent configuration across all 
nodes and restart Accumulo.

I think you have fooled yourself that the second exception is due to you 
reading from a user table. Note again, the user in question is the 
!SYSTEM user. You are not this user.

Webb, Ryan L. wrote:
> Below are two stack traces – one is from the GC and the latter is form
> the monitor when I try to look at a specific table.
>
> I am using 1.6.0-CHD5.1.0 on CentOS 6.4
>
> This happened after I attempted to fix the FQDN issue with bulk table
> loading.
>
> Is there a way to reset the password? How did this get changed?
>
> Unable to scan metadata table
>
> java.lang.RuntimeException:
> org.apache.accumulo.core.client.AccumuloSecurityException: Error
> BAD_CREDENTIALS for user !SYSTEM on table accumulo.root(ID:+r) -
> Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.ScannerIterator.hasNext(ScannerIterator.java:187)
>
> at
> org.apache.accumulo.server.util.MetadataTableUtil.getLogEntries(MetadataTableUtil.java:547)
>
> at
> org.apache.accumulo.server.util.MetadataTableUtil$LogEntryIterator.<init>(MetadataTableUtil.java:615)
>
> at
> org.apache.accumulo.server.util.MetadataTableUtil.getLogEntries(MetadataTableUtil.java:652)
>
> at
> org.apache.accumulo.gc.GarbageCollectWriteAheadLogs.removeMetadataEntries(GarbageCollectWriteAheadLogs.java:280)
>
> at
> org.apache.accumulo.gc.GarbageCollectWriteAheadLogs.collect(GarbageCollectWriteAheadLogs.java:125)
>
> at
> org.apache.accumulo.gc.SimpleGarbageCollector.run(SimpleGarbageCollector.java:568)
>
> at
> org.apache.accumulo.gc.SimpleGarbageCollector.main(SimpleGarbageCollector.java:154)
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:606)
>
> at org.apache.accumulo.start.Main$1.run(Main.java:141)
>
> at java.lang.Thread.run(Thread.java:744)
>
> Caused by: org.apache.accumulo.core.client.AccumuloSecurityException:
> Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.root(ID:+r) -
> Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.java:454)
>
> at
> org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.java:277)
>
> at
> org.apache.accumulo.core.client.impl.ScannerIterator$Reader.run(ScannerIterator.java:84)
>
> at
> org.apache.accumulo.core.client.impl.ScannerIterator.hasNext(ScannerIterator.java:177)
>
> ... 13 more
>
> Caused by: ThriftSecurityException(user:!SYSTEM, code:BAD_CREDENTIALS)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startScan_result$startScan_resultStandardScheme.read(TabletClientService.java:5100)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startScan_result$startScan_resultStandardScheme.read(TabletClientService.java:5077)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startScan_result.read(TabletClientService.java:5000)
>
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.recv_startScan(TabletClientService.java:226)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.startScan(TabletClientService.java:202)
>
> at
> org.apache.accumulo.core.client.impl.ThriftScanner.scan(ThriftScanner.java:399)
>
> ... 16 more
>
> java.lang.RuntimeException:
> org.apache.accumulo.core.client.AccumuloSecurityException: Error
> BAD_CREDENTIALS for user !SYSTEM on table accumulo.metadata(ID:!0) -
> Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.hasNext(TabletServerBatchReaderIterator.java:208)
>
> at
> org.apache.accumulo.server.master.state.MetaDataTableScanner.hasNext(MetaDataTableScanner.java:117)
>
> at
> org.apache.accumulo.monitor.servlets.TablesServlet.doTableDetails(TablesServlet.java:158)
>
> at
> org.apache.accumulo.monitor.servlets.TablesServlet.pageBody(TablesServlet.java:75)
>
> at
> org.apache.accumulo.monitor.servlets.BasicServlet.doGet(BasicServlet.java:65)
>
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
>
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>
> at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
>
> at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:401)
>
> at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
>
> at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
>
> at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:326)
>
> at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
>
> at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
>
> at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
>
> at
> org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
>
> Caused by: org.apache.accumulo.core.client.AccumuloSecurityException:
> Error BAD_CREDENTIALS for user !SYSTEM on table accumulo.metadata(ID:!0)
> - Username or Password is Invalid
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.doLookup(TabletServerBatchReaderIterator.java:716)
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator$QueryTask.run(TabletServerBatchReaderIterator.java:372)
>
> at
> org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
>
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>
> at
> org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
>
> at
> org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
>
> at java.lang.Thread.run(Thread.java:744)
>
> Caused by: ThriftSecurityException(user:!SYSTEM, code:BAD_CREDENTIALS)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startMultiScan_result$startMultiScan_resultStandardScheme.read(TabletClientService.java:8613)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startMultiScan_result$startMultiScan_resultStandardScheme.read(TabletClientService.java:8590)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$startMultiScan_result.read(TabletClientService.java:8529)
>
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.recv_startMultiScan(TabletClientService.java:311)
>
> at
> org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Client.startMultiScan(TabletClientService.java:291)
>
> at
> org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.doLookup(TabletServerBatchReaderIterator.java:658)
>
> ... 7 more
>