[jira] [Updated] (TEZ-2889) ATSHistoryACLPolicyManager makes DAG Submission wait on ATS requests

["Hitesh Shah (JIRA)" <ji...@apache.org>]

     [ https://issues.apache.org/jira/browse/TEZ-2889?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Hitesh Shah updated TEZ-2889:
-----------------------------
    Assignee:     (was: Hitesh Shah)

> ATSHistoryACLPolicyManager makes DAG Submission wait on ATS requests
> --------------------------------------------------------------------
>
>                 Key: TEZ-2889
>                 URL: https://issues.apache.org/jira/browse/TEZ-2889
>             Project: Apache Tez
>          Issue Type: Bug
>    Affects Versions: 0.7.0, 0.8.2
>            Reporter: Gopal V
>              Labels: ATS, Performance
>
> On a cluster with a broken ATS install, the ATS domain creation is in-thread & bottlenecks all Tez queries, reducing the cluster throughput by ~3x for very small queries.
> {code}
> java.net.SocketInputStream.socketRead0(FileDescriptor, byte[], int, int, int) SocketInputStream.java (native)
> java.net.SocketInputStream.socketRead(FileDescriptor, byte[], int, int, int) SocketInputStream.java:116
> java.net.SocketInputStream.read(byte[], int, int, int) SocketInputStream.java:170
> java.net.SocketInputStream.read(byte[], int, int) SocketInputStream.java:141
> java.io.BufferedInputStream.fill() BufferedInputStream.java:246
> java.io.BufferedInputStream.read1(byte[], int, int) BufferedInputStream.java:286
> java.io.BufferedInputStream.read(byte[], int, int) BufferedInputStream.java:345
> sun.net.www.http.HttpClient.parseHTTPHeader(MessageHeader, ProgressSource, HttpURLConnection) HttpClient.java:704
> sun.net.www.http.HttpClient.parseHTTP(MessageHeader, ProgressSource, HttpURLConnection) HttpClient.java:647
> sun.net.www.protocol.http.HttpURLConnection.getInputStream0() HttpURLConnection.java:1535
> sun.net.www.protocol.http.HttpURLConnection.getInputStream() HttpURLConnection.java:1440
> java.net.HttpURLConnection.getResponseCode() HttpURLConnection.java:480
> org.apache.hadoop.security.authentication.client.AuthenticatedURL.extractToken(HttpURLConnection, AuthenticatedURL$Token) AuthenticatedURL.java:252
> org.apache.hadoop.security.authentication.client.PseudoAuthenticator.authenticate(URL, AuthenticatedURL$Token) PseudoAuthenticator.java:77
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(URL, AuthenticatedURL$Token) DelegationTokenAuthenticator.java:128
> org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(URL, AuthenticatedURL$Token) AuthenticatedURL.java:215
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.openConnection(URL, DelegationTokenAuthenticatedURL$Token, String) DelegationTokenAuthenticatedURL.java:322
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineURLConnectionFactory.getHttpURLConnection(URL) TimelineClientImpl.java:493
> com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(ClientRequest) URLConnectionClientHandler.java:159
> com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(ClientRequest) URLConnectionClientHandler.java:147
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter$1.run() TimelineClientImpl.java:234
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineClientConnectionRetry.retryOn(TimelineClientImpl$TimelineClientRetryOp) TimelineClientImpl.java:183
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter.handle(ClientRequest) TimelineClientImpl.java:245
> com.sun.jersey.api.client.Client.handle(ClientRequest) Client.java:648
> com.sun.jersey.api.client.WebResource.handle(Class, ClientRequest) WebResource.java:670
> com.sun.jersey.api.client.WebResource.access$200(WebResource, Class, ClientRequest) WebResource.java:74
> com.sun.jersey.api.client.WebResource$Builder.put(Class, Object) WebResource.java:533
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPostingObject(Object, String) TimelineClientImpl.java:478
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run() TimelineClientImpl.java:323
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run() TimelineClientImpl.java:320
> java.security.AccessController.doPrivileged(PrivilegedExceptionAction, AccessControlContext) AccessController.java (native)
> javax.security.auth.Subject.doAs(Subject, PrivilegedExceptionAction) Subject.java:422
> org.apache.hadoop.security.UserGroupInformation.doAs(PrivilegedExceptionAction) UserGroupInformation.java:1657
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPosting(Object, String) TimelineClientImpl.java:320
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.putDomain(TimelineDomain) TimelineClientImpl.java:313
> org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(String, Configuration, DAGAccessControls) ATSHistoryACLPolicyManager.java:129
> org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createDAGDomain(Configuration, ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:207
> org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionDAGACLs(Configuration, ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:240
> org.apache.tez.client.TezClient.submitDAGSession(DAG) TezClient.java:491
> org.apache.tez.client.TezClient.submitDAG(DAG) TezClient.java:452
> org.apache.hadoop.hive.ql.exec.tez.TezTask.submit(JobConf, DAG, Path, LocalResource, TezSessionState, List, String[], Map) TezTask.java:427
> org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(DriverContext) TezTask.java:170
> org.apache.hadoop.hive.ql.exec.Task.executeTask() Task.java:160
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential() TaskRunner.java:89
> org.apache.hadoop.hive.ql.Driver.launchTask(Task, String, boolean, String, int, DriverContext) Driver.java:1758
> org.apache.hadoop.hive.ql.Driver.execute() Driver.java:1517
> org.apache.hadoop.hive.ql.Driver.runInternal(String, boolean) Driver.java:1275
> org.apache.hadoop.hive.ql.Driver.run(String, boolean) Driver.java:1091
> org.apache.hadoop.hive.ql.Driver.run(String) Driver.java:1081
> org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(String, CommandProcessor, CliSessionState) CliDriver.java:225
> org.apache.hadoop.hive.cli.CliDriver.processCmd(String) CliDriver.java:177
> org.apache.hadoop.hive.cli.CliDriver.processLine(String, boolean) CliDriver.java:388
> org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliSessionState, HiveConf, OptionsProcessor) CliDriver.java:759
> org.apache.hadoop.hive.cli.CliDriver.run(String[]) CliDriver.java:704
> org.apache.hadoop.hive.cli.CliDriver.main(String[]) CliDriver.java:633
> sun.reflect.NativeMethodAccessorImpl.invoke0(Method, Object, Object[]) NativeMethodAccessorImpl.java (native)
> sun.reflect.NativeMethodAccessorImpl.invoke(Object, Object[]) NativeMethodAccessorImpl.java:62
> sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[]) DelegatingMethodAccessorImpl.java:43
> java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
> org.apache.hadoop.util.RunJar.run(String[]) RunJar.java:221
> org.apache.hadoop.util.RunJar.main(String[]) RunJar.java:136
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)