You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "ASF GitHub Bot (Jira)" <ji...@apache.org> on 2022/06/20 03:14:00 UTC
[jira] [Updated] (HIVE-26045) Detect timed out connections for providers and auto-reconnect
[ https://issues.apache.org/jira/browse/HIVE-26045?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
ASF GitHub Bot updated HIVE-26045:
----------------------------------
Labels: pull-request-available (was: )
> Detect timed out connections for providers and auto-reconnect
> -------------------------------------------------------------
>
> Key: HIVE-26045
> URL: https://issues.apache.org/jira/browse/HIVE-26045
> Project: Hive
> Issue Type: Sub-task
> Components: HiveServer2
> Affects Versions: 4.0.0
> Reporter: Naveen Gangam
> Assignee: zhangbutao
> Priority: Major
> Labels: pull-request-available
> Time Spent: 10m
> Remaining Estimate: 0h
>
> For the connectors, we use single connection, no pooling. But when the connection is idle for an extended period, the JDBC connection times out. We need to check for closed connections (Connection.isClosed()?) and re-establish the connection. Otherwise it renders the connector fairly useless.
> {noformat}
> 2022-03-17T13:02:16,635 WARN [HiveServer2-Handler-Pool: Thread-116] thrift.ThriftCLIService: Error executing statement:
> org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: SemanticException Unable to fetch table temp_dbs. Error retrieving remote table:com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: No operations allowed after connection closed.
> at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:373) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:211) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:265) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.operation.Operation.run(Operation.java:285) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:576) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:562) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?]
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_231]
> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_231]
> at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_231]
> at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_231]
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1682) ~[hadoop-common-3.1.0.jar:?]
> at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at com.sun.proxy.$Proxy44.executeStatementAsync(Unknown Source) ~[?:?]
> at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:315) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:567) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1550) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.rpc.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1530) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:38) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:313) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_231]
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_231]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_231]
> Caused by: org.apache.hadoop.hive.ql.parse.SemanticException: Unable to fetch table temp_dbs. Error retrieving remote table:com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: No operations allowed after connection closed.
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1821) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1799) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1795) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableAnalyzer.analyzeInternal(DescTableAnalyzer.java:76) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:306) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:223) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:104) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:196) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:615) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:561) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:555) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:127) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:204) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> ... 26 more
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to fetch table temp_dbs. Error retrieving remote table:com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: No operations allowed after connection closed.
> at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1563) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1511) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1491) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1815) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1799) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1795) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableAnalyzer.analyzeInternal(DescTableAnalyzer.java:76) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:306) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:223) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:104) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:196) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:615) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:561) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:555) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:127) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:204) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> ... 26 more
> Caused by: org.apache.hadoop.hive.metastore.api.MetaException: Error retrieving remote table:com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: No operations allowed after connection closed.
> at org.apache.hadoop.hive.metastore.dataconnector.jdbc.AbstractJDBCConnectorProvider.getTable(AbstractJDBCConnectorProvider.java:231) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getTableInternal(HiveMetaStore.java:3831) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_table_req(HiveMetaStore.java:3798) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at sun.reflect.GeneratedMethodAccessor64.invoke(Unknown Source) ~[?:?]
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_231]
> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_231]
> at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:160) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:121) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at com.sun.proxy.$Proxy40.get_table_req(Unknown Source) ~[?:?]
> at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTableInternal(HiveMetaStoreClient.java:2341) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.metastore.HiveMetaStoreClientWithLocalCache.getTableInternal(HiveMetaStoreClientWithLocalCache.java:339) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTableInternal(SessionHiveMetaStoreClient.java:2033) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:2369) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTable(SessionHiveMetaStoreClient.java:260) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTable(SessionHiveMetaStoreClient.java:246) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?]
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_231]
> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_231]
> at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:216) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at com.sun.proxy.$Proxy41.getTable(Unknown Source) ~[?:?]
> at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1554) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1511) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1491) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1815) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1799) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getTable(BaseSemanticAnalyzer.java:1795) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableAnalyzer.analyzeInternal(DescTableAnalyzer.java:76) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:306) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:223) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:104) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:196) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:615) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:561) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:555) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:127) ~[hive-exec-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:204) ~[hive-service-3.1.3000.7.2.15.0-SNAPSHOT.jar:3.1.3000.7.2.15.0-SNAPSHOT]
> ... 26 more
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)