You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2020/10/01 02:37:52 UTC
[GitHub] [spark] AngersZhuuuu edited a comment on pull request #29881: [SPARK-32852][SQL] spark.sql.hive.metastore.jars support HDFS location
AngersZhuuuu edited a comment on pull request #29881:
URL: https://github.com/apache/spark/pull/29881#issuecomment-701211241
> Hi, @AngersZhuuuu . Is this PR limited to HDFS? What about `S3` storage?
Hi @dongjoon-hyun , when I use `s3` path, it can pass check and get listed file paths, but I am confused about this `StackOverflowError`, it seems not normal fetch `s3` data file error, but a error when ClassLoader fetch `s3` URL.
I am not so familiar with `s3` ,can you give some advise?
if we use `Utils.fetchFIle` and return a local path seems will be ok.
```
Exception in thread "main" java.lang.StackOverflowError
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath.getLoader(URLClassPath.java:554)
at sun.misc.URLClassPath.getLoader(URLClassPath.java:519)
at sun.misc.URLClassPath.getNextLoader(URLClassPath.java:484)
at sun.misc.URLClassPath.access$100(URLClassPath.java:65)
at sun.misc.URLClassPath$1.next(URLClassPath.java:266)
at sun.misc.URLClassPath$1.hasMoreElements(URLClassPath.java:277)
at java.net.URLClassLoader$3$1.run(URLClassLoader.java:601)
at java.net.URLClassLoader$3$1.run(URLClassLoader.java:599)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader$3.next(URLClassLoader.java:598)
at java.net.URLClassLoader$3.hasMoreElements(URLClassLoader.java:623)
at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getConfigurationFile(LogFactory.java:1409)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getFactory(LogFactory.java:455)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getLog(LogFactory.java:657)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParser.<init>(DefaultHttpResponseParser.java:59)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParserFactory.create(DefaultHttpResponseParserFactory.java:76)
at com.amazonaws.thirdparty.apache.http.impl.DefaultBHttpClientConnection.<init>(DefaultBHttpClientConnection.java:99)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultManagedHttpClientConnection.<init>(DefaultManagedHttpClientConnection.java:74)
at com.amazonaws.thirdparty.apache.http.impl.conn.LoggingManagedHttpClientConnection.<init>(LoggingManagedHttpClientConnection.java:66)
at com.amazonaws.thirdparty.apache.http.impl.conn.ManagedHttpClientConnectionFactory.create(ManagedHttpClientConnectionFactory.java:127)
at com.amazonaws.thirdparty.apache.http.impl.conn.ManagedHttpClientConnectionFactory.create(ManagedHttpClientConnectionFactory.java:57)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$InternalConnectionFactory.create(PoolingHttpClientConnectionManager.java:619)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$InternalConnectionFactory.create(PoolingHttpClientConnectionManager.java:590)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:363)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool.access$200(AbstractConnPool.java:69)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:246)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:193)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.java:303)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:279)
at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.amazonaws.http.conn.ClientConnectionRequestFactory$Handler.invoke(ClientConnectionRequestFactory.java:70)
at com.amazonaws.http.conn.$Proxy27.get(Unknown Source)
at com.amazonaws.thirdparty.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:191)
at com.amazonaws.thirdparty.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1238)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1058)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4368)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4315)
at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1271)
at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getObjectMetadata$4(S3AFileSystem.java:1290)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:285)
at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:1287)
at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:2224)
at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:2204)
at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:2143)
at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:716)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:899)
at org.apache.hadoop.fs.FsUrlConnection.connect(FsUrlConnection.java:60)
at org.apache.hadoop.fs.FsUrlConnection.getInputStream(FsUrlConnection.java:69)
at sun.net.www.protocol.jar.URLJarFile.retrieve(URLJarFile.java:214)
at sun.net.www.protocol.jar.URLJarFile.getJarFile(URLJarFile.java:71)
at sun.net.www.protocol.jar.JarFileFactory.get(JarFileFactory.java:84)
at sun.net.www.protocol.jar.JarURLConnection.connect(JarURLConnection.java:122)
at sun.net.www.protocol.jar.JarURLConnection.getJarFile(JarURLConnection.java:89)
at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:934)
at sun.misc.URLClassPath$JarLoader.access$800(URLClassPath.java:791)
at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:876)
at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:869)
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:868)
at sun.misc.URLClassPath$JarLoader.<init>(URLClassPath.java:819)
at sun.misc.URLClassPath$3.run(URLClassPath.java:565)
at sun.misc.URLClassPath$3.run(URLClassPath.java:555)
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath.getLoader(URLClassPath.java:554)
at sun.misc.URLClassPath.getLoader(URLClassPath.java:519)
at sun.misc.URLClassPath.getNextLoader(URLClassPath.java:484)
at sun.misc.URLClassPath.access$100(URLClassPath.java:65)
at sun.misc.URLClassPath$1.next(URLClassPath.java:266)
at sun.misc.URLClassPath$1.hasMoreElements(URLClassPath.java:277)
at java.net.URLClassLoader$3$1.run(URLClassLoader.java:601)
at java.net.URLClassLoader$3$1.run(URLClassLoader.java:599)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader$3.next(URLClassLoader.java:598)
at java.net.URLClassLoader$3.hasMoreElements(URLClassLoader.java:623)
at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getConfigurationFile(LogFactory.java:1409)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getFactory(LogFactory.java:455)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getLog(LogFactory.java:657)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParser.<init>(DefaultHttpResponseParser.java:59)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParserFactory.create(DefaultHttpResponseParserFactory.java:76)
at com.amazonaws.thirdparty.apache.http.impl.DefaultBHttpClientConnection.<init>(DefaultBHttpClientConnection.java:99)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultManagedHttpClientConnection.<init>(DefaultManagedHttpClientConnection.java:74)
at com.amazonaws.thirdparty.apache.http.impl.conn.LoggingManagedHttpClientConnection.<init>(LoggingManagedHttpClientConnection.java:66)
at com.amazonaws.thirdparty.apache.http.impl.conn.ManagedHttpClientConnectionFactory.create(ManagedHttpClientConnectionFactory.java:127)
at com.amazonaws.thirdparty.apache.http.impl.conn.ManagedHttpClientConnectionFactory.create(ManagedHttpClientConnectionFactory.java:57)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$InternalConnectionFactory.create(PoolingHttpClientConnectionManager.java:619)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$InternalConnectionFactory.create(PoolingHttpClientConnectionManager.java:590)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:363)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool.access$200(AbstractConnPool.java:69)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:246)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:193)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.java:303)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:279)
at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.amazonaws.http.conn.ClientConnectionRequestFactory$Handler.invoke(ClientConnectionRequestFactory.java:70)
at com.amazonaws.http.conn.$Proxy27.get(Unknown Source)
at com.amazonaws.thirdparty.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:191)
at com.amazonaws.thirdparty.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1238)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1058)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4368)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4315)
at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1271)
at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getObjectMetadata$4(S3AFileSystem.java:1290)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:285)
at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:1287)
at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:2224)
at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:2204)
at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:2143)
at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:716)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:899)
at org.apache.hadoop.fs.FsUrlConnection.connect(FsUrlConnection.java:60)
at org.apache.hadoop.fs.FsUrlConnection.getInputStream(FsUrlConnection.java:69)
at sun.net.www.protocol.jar.URLJarFile.retrieve(URLJarFile.java:214)
at sun.net.www.protocol.jar.URLJarFile.getJarFile(URLJarFile.java:71)
at sun.net.www.protocol.jar.JarFileFactory.get(JarFileFactory.java:84)
at sun.net.www.protocol.jar.JarURLConnection.connect(JarURLConnection.java:122)
at sun.net.www.protocol.jar.JarURLConnection.getJarFile(JarURLConnection.java:89)
at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:934)
at sun.misc.URLClassPath$JarLoader.access$800(URLClassPath.java:791)
at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:876)
at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:869)
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:868)
at sun.misc.URLClassPath$JarLoader.<init>(URLClassPath.java:819)
at sun.misc.URLClassPath$3.run(URLClassPath.java:565)
at sun.misc.URLClassPath$3.run(URLClassPath.java:555)
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath.getLoader(URLClassPath.java:554)
at sun.misc.URLClassPath.getLoader(URLClassPath.java:519)
at sun.misc.URLClassPath.getNextLoader(URLClassPath.java:484)
at sun.misc.URLClassPath.access$100(URLClassPath.java:65)
at sun.misc.URLClassPath$1.next(URLClassPath.java:266)
at sun.misc.URLClassPath$1.hasMoreElements(URLClassPath.java:277)
at java.net.URLClassLoader$3$1.run(URLClassLoader.java:601)
at java.net.URLClassLoader$3$1.run(URLClassLoader.java:599)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader$3.next(URLClassLoader.java:598)
at java.net.URLClassLoader$3.hasMoreElements(URLClassLoader.java:623)
at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getConfigurationFile(LogFactory.java:1409)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getFactory(LogFactory.java:455)
at com.amazonaws.thirdparty.apache.logging.LogFactory.getLog(LogFactory.java:657)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParser.<init>(DefaultHttpResponseParser.java:59)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParserFactory.create(DefaultHttpResponseParserFactory.java:76)
at com.amazonaws.thirdparty.apache.http.impl.DefaultBHttpClientConnection.<init>(DefaultBHttpClientConnection.java:99)
at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultManagedHttpClientConnection.<init>(DefaultManagedHttpClientConnection.java:74)
at com.amazonaws.thirdparty.apache.http.impl.conn.LoggingManagedHttpClientConnection.<init>(LoggingManagedHttpClientConnection.java:66)
at com.amazonaws.thirdparty.apache.http.impl.conn.ManagedHttpClientConnectionFactory.create(ManagedHttpClientConnectionFactory.java:127)
at com.amazonaws.thirdparty.apache.http.impl.conn.ManagedHttpClientConnectionFactory.create(ManagedHttpClientConnectionFactory.java:57)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$InternalConnectionFactory.create(PoolingHttpClientConnectionManager.java:619)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$InternalConnectionFactory.create(PoolingHttpClientConnectionManager.java:590)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:363)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool.access$200(AbstractConnPool.java:69)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:246)
at com.amazonaws.thirdparty.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:193)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.java:303)
at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:279)
at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.amazonaws.http.conn.ClientConnectionRequestFactory$Handler.invoke(ClientConnectionRequestFactory.java:70)
at com.amazonaws.http.conn.$Proxy27.get(Unknown Source)
at com.amazonaws.thirdparty.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:191)
at com.amazonaws.thirdparty.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1238)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1058)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org