You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/02/21 03:37:03 UTC

svn commit: r1661271 [39/39] - in /hive/branches/llap: ./ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/ accumulo...

Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Sat Feb 21 02:36:54 2015
@@ -20,7 +20,7 @@ package org.apache.hive.service.cli.thri
 
 import java.util.Arrays;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
@@ -63,9 +63,10 @@ public class ThriftHttpCLIService extend
       httpServer = new org.eclipse.jetty.server.Server();
 
       // Server thread pool
+      // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
       String threadPoolName = "HiveServer2-HttpHandler-Pool";
       ExecutorService executorService = new ThreadPoolExecutor(minWorkerThreads, maxWorkerThreads,
-          workerKeepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+          workerKeepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
           new ThreadFactoryWithGarbageCleanup(threadPoolName));
       ExecutorThreadPool threadPool = new ExecutorThreadPool(executorService);
       httpServer.setThreadPool(threadPool);

Modified: hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Sat Feb 21 02:36:54 2015
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUti
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.Options;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
 import org.apache.hadoop.crypto.key.KeyProviderFactory;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.DefaultFileAccess;
@@ -511,8 +512,10 @@ public class Hadoop23Shims extends Hadoo
 
     // Need to set the client's KeyProvider to the NN's for JKS,
     // else the updates do not get flushed properly
-    miniDFSCluster.getFileSystem().getClient().setKeyProvider(
-        miniDFSCluster.getNameNode().getNamesystem().getProvider());
+    KeyProviderCryptoExtension keyProvider =  miniDFSCluster.getNameNode().getNamesystem().getProvider();
+    if (keyProvider != null) {
+      miniDFSCluster.getFileSystem().getClient().setKeyProvider(keyProvider);
+    }
 
     cluster = new MiniDFSShim(miniDFSCluster);
     return cluster;

Modified: hive/branches/llap/shims/common/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/common/pom.xml?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/shims/common/pom.xml (original)
+++ hive/branches/llap/shims/common/pom.xml Sat Feb 21 02:36:54 2015
@@ -74,7 +74,6 @@
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-framework</artifactId>
-       <version>${curator.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.zookeeper</groupId>

Modified: hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java (original)
+++ hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java Sat Feb 21 02:36:54 2015
@@ -18,7 +18,7 @@
 package org.apache.hive.spark.client;
 
 import java.io.Serializable;
-import java.net.URL;
+import java.net.URI;
 import java.util.concurrent.Future;
 
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
@@ -68,10 +68,10 @@ public interface SparkClient extends Ser
    * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
    * on that node (and not on the client machine).
    *
-   * @param url The location of the jar file.
+   * @param uri The location of the jar file.
    * @return A future that can be used to monitor the operation.
    */
-  Future<?> addJar(URL url);
+  Future<?> addJar(URI uri);
 
   /**
    * Adds a file to the running remote context.
@@ -80,10 +80,10 @@ public interface SparkClient extends Ser
    * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist
    * on that node (and not on the client machine).
    *
-   * @param url The location of the file.
+   * @param uri The location of the file.
    * @return A future that can be used to monitor the operation.
    */
-  Future<?> addFile(URL url);
+  Future<?> addFile(URI uri);
 
   /**
    * Get the count of executors.

Modified: hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java (original)
+++ hive/branches/llap/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java Sat Feb 21 02:36:54 2015
@@ -17,6 +17,14 @@
 
 package org.apache.hive.spark.client;
 
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.util.concurrent.GenericFutureListener;
 import io.netty.util.concurrent.Promise;
@@ -30,14 +38,12 @@ import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.io.Serializable;
 import java.io.Writer;
-import java.net.URL;
+import java.net.URI;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.UUID;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -49,14 +55,6 @@ import org.apache.spark.SparkException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Charsets;
-import com.google.common.base.Joiner;
-import com.google.common.base.Strings;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
 class SparkClientImpl implements SparkClient {
   private static final long serialVersionUID = 1L;
 
@@ -65,6 +63,8 @@ class SparkClientImpl implements SparkCl
   private static final long DEFAULT_SHUTDOWN_TIMEOUT = 10000; // In milliseconds
 
   private static final String OSX_TEST_OPTS = "SPARK_OSX_TEST_OPTS";
+  private static final String SPARK_HOME_ENV = "SPARK_HOME";
+  private static final String SPARK_HOME_KEY = "spark.home";
   private static final String DRIVER_OPTS_KEY = "spark.driver.extraJavaOptions";
   private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions";
   private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath";
@@ -153,13 +153,13 @@ class SparkClientImpl implements SparkCl
   }
 
   @Override
-  public Future<?> addJar(URL url) {
-    return run(new AddJarJob(url.toString()));
+  public Future<?> addJar(URI uri) {
+    return run(new AddJarJob(uri.toString()));
   }
 
   @Override
-  public Future<?> addFile(URL url) {
-    return run(new AddFileJob(url.toString()));
+  public Future<?> addFile(URI uri) {
+    return run(new AddFileJob(uri.toString()));
   }
 
   @Override
@@ -213,9 +213,12 @@ class SparkClientImpl implements SparkCl
       // If a Spark installation is provided, use the spark-submit script. Otherwise, call the
       // SparkSubmit class directly, which has some caveats (like having to provide a proper
       // version of Guava on the classpath depending on the deploy mode).
-      String sparkHome = conf.get("spark.home");
+      String sparkHome = conf.get(SPARK_HOME_KEY);
+      if (sparkHome == null) {
+        sparkHome = System.getenv(SPARK_HOME_ENV);
+      }
       if (sparkHome == null) {
-        sparkHome = System.getProperty("spark.home");
+        sparkHome = System.getProperty(SPARK_HOME_KEY);
       }
       String sparkLogDir = conf.get("hive.spark.log.dir");
       if (sparkLogDir == null) {

Modified: hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java (original)
+++ hive/branches/llap/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java Sat Feb 21 02:36:54 2015
@@ -22,7 +22,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.io.Serializable;
-import java.net.URL;
+import java.net.URI;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -204,7 +204,7 @@ public class TestSparkClient {
           jarFile.closeEntry();
           jarFile.close();
 
-          client.addJar(new URL("file:" + jar.getAbsolutePath()))
+          client.addJar(new URI("file:" + jar.getAbsolutePath()))
             .get(TIMEOUT, TimeUnit.SECONDS);
 
           // Need to run a Spark job to make sure the jar is added to the class loader. Monitoring
@@ -220,7 +220,7 @@ public class TestSparkClient {
           fileStream.write("test file".getBytes("UTF-8"));
           fileStream.close();
 
-          client.addJar(new URL("file:" + file.getAbsolutePath()))
+          client.addJar(new URI("file:" + file.getAbsolutePath()))
             .get(TIMEOUT, TimeUnit.SECONDS);
 
           // The same applies to files added with "addFile". They're only guaranteed to be available