You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/02/21 03:37:03 UTC
svn commit: r1661271 [39/39] - in /hive/branches/llap: ./
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/
accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/
accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/ accumulo...
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Sat Feb 21 02:36:54 2015
@@ -20,7 +20,7 @@ package org.apache.hive.service.cli.thri
import java.util.Arrays;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -63,9 +63,10 @@ public class ThriftHttpCLIService extend
httpServer = new org.eclipse.jetty.server.Server();
// Server thread pool
+ // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
String threadPoolName = "HiveServer2-HttpHandler-Pool";
ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
- workerKeepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new ThreadFactoryWithGarbageCleanup(threadPoolName));
ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
httpServer.setThreadPool(threadPool);
Modified: hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Sat Feb 21 02:36:54 2015
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUti
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.Options;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.DefaultFileAccess;
@@ -511,8 +512,10 @@ public class Hadoop23Shims extends Hadoo
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
- miniDFSCluster.getFileSystem().getClient().setKeyProvider(
- miniDFSCluster.getNameNode().getNamesystem().getProvider());
+ KeyProviderCryptoExtension keyProvider = miniDFSCluster.getNameNode().getNamesystem().getProvider();
+ if (keyProvider != null) {
+ miniDFSCluster.getFileSystem().getClient().setKeyProvider(keyProvider);
+ }
cluster = new MiniDFSShim(miniDFSCluster);
return cluster;
Modified: hive/branches/llap/shims/common/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/common/pom.xml?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/shims/common/pom.xml (original)
+++ hive/branches/llap/shims/common/pom.xml Sat Feb 21 02:36:54 2015
@@ -74,7 +74,6 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
Modified: hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java (original)
+++ hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java Sat Feb 21 02:36:54 2015
@@ -18,7 +18,7 @@
package org.apache.hive.spark.client;
import java.io.Serializable;
-import java.net.URL;
+import java.net.URI;
import java.util.concurrent.Future;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
@@ -68,10 +68,10 @@ public interface SparkClient extends Ser
* in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
* on that node (and not on the client machine).
*
- * @param url The location of the jar file.
+ * @param uri The location of the jar file.
* @return A future that can be used to monitor the operation.
*/
- Future<?> addJar(URL url);
+ Future<?> addJar(URI uri);
/**
* Adds a file to the running remote context.
@@ -80,10 +80,10 @@ public interface SparkClient extends Ser
* in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
* on that node (and not on the client machine).
*
- * @param url The location of the file.
+ * @param uri The location of the file.
* @return A future that can be used to monitor the operation.
*/
- Future<?> addFile(URL url);
+ Future<?> addFile(URI uri);
/**
* Get the count of executors.
Modified: hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java (original)
+++ hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java Sat Feb 21 02:36:54 2015
@@ -17,6 +17,14 @@
package org.apache.hive.spark.client;
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
import io.netty.channel.ChannelHandlerContext;
import io.netty.util.concurrent.GenericFutureListener;
import io.netty.util.concurrent.Promise;
@@ -30,14 +38,12 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.io.Writer;
-import java.net.URL;
+import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -49,14 +55,6 @@ import org.apache.spark.SparkException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Charsets;
-import com.google.common.base.Joiner;
-import com.google.common.base.Strings;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
class SparkClientImpl implements SparkClient {
private static final long serialVersionUID = 1L;
@@ -65,6 +63,8 @@ class SparkClientImpl implements SparkCl
private static final long DEFAULT_SHUTDOWN_TIMEOUT = 10000; // In milliseconds
private static final String OSX_TEST_OPTS = "SPARK_OSX_TEST_OPTS";
+ private static final String SPARK_HOME_ENV = "SPARK_HOME";
+ private static final String SPARK_HOME_KEY = "spark.home";
private static final String DRIVER_OPTS_KEY = "spark.driver.extraJavaOptions";
private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions";
private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath";
@@ -153,13 +153,13 @@ class SparkClientImpl implements SparkCl
}
@Override
- public Future<?> addJar(URL url) {
- return run(new AddJarJob(url.toString()));
+ public Future<?> addJar(URI uri) {
+ return run(new AddJarJob(uri.toString()));
}
@Override
- public Future<?> addFile(URL url) {
- return run(new AddFileJob(url.toString()));
+ public Future<?> addFile(URI uri) {
+ return run(new AddFileJob(uri.toString()));
}
@Override
@@ -213,9 +213,12 @@ class SparkClientImpl implements SparkCl
// If a Spark installation is provided, use the spark-submit script. Otherwise, call the
// SparkSubmit class directly, which has some caveats (like having to provide a proper
// version of Guava on the classpath depending on the deploy mode).
- String sparkHome = conf.get("spark.home");
+ String sparkHome = conf.get(SPARK_HOME_KEY);
+ if (sparkHome == null) {
+ sparkHome = System.getenv(SPARK_HOME_ENV);
+ }
if (sparkHome == null) {
- sparkHome = System.getProperty("spark.home");
+ sparkHome = System.getProperty(SPARK_HOME_KEY);
}
String sparkLogDir = conf.get("hive.spark.log.dir");
if (sparkLogDir == null) {
Modified: hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java (original)
+++ hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java Sat Feb 21 02:36:54 2015
@@ -22,7 +22,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.Serializable;
-import java.net.URL;
+import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -204,7 +204,7 @@ public class TestSparkClient {
jarFile.closeEntry();
jarFile.close();
- client.addJar(new URL("file:" + jar.getAbsolutePath()))
+ client.addJar(new URI("file:" + jar.getAbsolutePath()))
.get(TIMEOUT, TimeUnit.SECONDS);
// Need to run a Spark job to make sure the jar is added to the class loader. Monitoring
@@ -220,7 +220,7 @@ public class TestSparkClient {
fileStream.write("test file".getBytes("UTF-8"));
fileStream.close();
- client.addJar(new URL("file:" + file.getAbsolutePath()))
+ client.addJar(new URI("file:" + file.getAbsolutePath()))
.get(TIMEOUT, TimeUnit.SECONDS);
// The same applies to files added with "addFile". They're only guaranteed to be available