You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/12/19 00:36:53 UTC
[2/2] hive git commit: HIVE-12633 : LLAP: package included serde jars
(Sergey Shelukhin/Gopal V, reviewed by Gopal V)
HIVE-12633 : LLAP: package included serde jars (Sergey Shelukhin/Gopal V, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4da7ed91
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4da7ed91
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4da7ed91
Branch: refs/heads/branch-2.0
Commit: 4da7ed913918792bd212109d8c4f72678136d979
Parents: 1d5e9c9
Author: Sergey Shelukhin <se...@apache.org>
Authored: Fri Dec 18 15:36:19 2015 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Fri Dec 18 15:36:35 2015 -0800
----------------------------------------------------------------------
.../hive/llap/cli/LlapOptionsProcessor.java | 16 +++++++-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 43 +++++++++++++++++++-
2 files changed, 55 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4da7ed91/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 8fd615c..58ef472 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -44,10 +44,11 @@ public class LlapOptionsProcessor {
private final long cache;
private final long size;
private final long xmx;
+ private final String jars;
private final Properties conf;
public LlapOptions(String name, int instances, String directory, int executors, long cache,
- long size, long xmx, @Nonnull Properties hiveconf) throws ParseException {
+ long size, long xmx, String jars, @Nonnull Properties hiveconf) throws ParseException {
if (instances <= 0) {
throw new ParseException("Invalid configuration: " + instances
+ " (should be greater than 0)");
@@ -59,6 +60,7 @@ public class LlapOptionsProcessor {
this.cache = cache;
this.size = size;
this.xmx = xmx;
+ this.jars = jars;
this.conf = hiveconf;
}
@@ -90,6 +92,10 @@ public class LlapOptionsProcessor {
return xmx;
}
+ public String getAuxJars() {
+ return jars;
+ }
+
public Properties getConfig() {
return conf;
}
@@ -134,6 +140,10 @@ public class LlapOptionsProcessor {
options.addOption(OptionBuilder.hasArg().withArgName("xmx").withLongOpt("xmx")
.withDescription("working memory size").create('w'));
+ options.addOption(OptionBuilder.hasArg().withArgName("auxjars").withLongOpt("auxjars")
+ .withDescription("additional jars to package (by default, JSON and HBase SerDe jars"
+ + " are packaged if available)").create('j'));
+
// -hiveconf x=y
options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value")
.withLongOpt("hiveconf").withDescription("Use value for given property").create());
@@ -156,6 +166,7 @@ public class LlapOptionsProcessor {
int instances = Integer.parseInt(commandLine.getOptionValue("instances"));
String directory = commandLine.getOptionValue("directory");
+ String jars = commandLine.getOptionValue("auxjars");
String name = commandLine.getOptionValue("name", null);
@@ -174,7 +185,8 @@ public class LlapOptionsProcessor {
// loglevel, chaosmonkey & args are parsed by the python processor
- return new LlapOptions(name, instances, directory, executors, cache, size, xmx, hiveconf);
+ return new LlapOptions(
+ name, instances, directory, executors, cache, size, xmx, jars, hiveconf);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4da7ed91/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 08d573b..8e5377f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.llap.cli;
import java.io.OutputStreamWriter;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -45,6 +47,9 @@ import com.google.common.base.Preconditions;
public class LlapServiceDriver {
protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName());
+ private static final String[] DEFAULT_AUX_CLASSES = new String[] {
+ "org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hadoop.hive.hbase.HBaseSerDe" };
+
private final Configuration conf;
public LlapServiceDriver() {
@@ -204,11 +209,45 @@ public class LlapServiceDriver {
CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), libDir.toString(), true);
lfs.delete(new Path(libDir, "tez.tar.gz"), false);
- // TODO: aux jars (like compression libs)
-
lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapInputFormat.class)), libDir);
lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(HiveInputFormat.class)), libDir);
+ // copy default aux classes (json/hbase)
+
+ for (String className : DEFAULT_AUX_CLASSES) {
+ String jarPath = null;
+ boolean hasException = false;
+ try {
+ Class<?> auxClass = Class.forName(className);
+ jarPath = Utilities.jarFinderGetJar(auxClass);
+ } catch (Throwable t) {
+ hasException = true;
+ String err =
+ "Cannot find a jar for [" + className + "] due to an exception (" + t.getMessage()
+ + "); not packaging the jar";
+ LOG.error(err, t);
+ System.err.println(err);
+ }
+ if (jarPath != null) {
+ lfs.copyFromLocalFile(new Path(jarPath), libDir);
+ } else if (!hasException) {
+ String err = "Cannot find a jar for [" + className + "]; not packaging the jar";
+ LOG.error(err);
+ System.err.println(err);
+ }
+ }
+
+ String auxJars = options.getAuxJars();
+ if (auxJars != null && !auxJars.isEmpty()) {
+ // TODO: transitive dependencies warning?
+ String[] jarPaths = auxJars.split(",");
+ for (String jarPath : jarPaths) {
+ if (!jarPath.isEmpty()) {
+ lfs.copyFromLocalFile(new Path(jarPath), libDir);
+ }
+ }
+ }
+
Path confPath = new Path(tmpDir, "conf");
lfs.mkdirs(confPath);