You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/06/22 18:15:35 UTC
hive git commit: HIVE-14057. Add an option to llapstatus to generate
output to a file, and minor bug fixes. (Siddharth Seth,
reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 67475129b -> f80f65da9
HIVE-14057. Add an option to llapstatus to generate output to a file, and minor bug fixes. (Siddharth Seth, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f80f65da
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f80f65da
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f80f65da
Branch: refs/heads/master
Commit: f80f65da9bbc03eea55ebc6ed726419dfda3d224
Parents: 6747512
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Jun 22 11:14:41 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Jun 22 11:14:41 2016 -0700
----------------------------------------------------------------------
.../llap/cli/LlapStatusOptionsProcessor.java | 38 +++++++---
.../hive/llap/cli/LlapStatusServiceDriver.java | 73 +++++++++++++++-----
.../java/org/apache/hive/http/LlapServlet.java | 3 +-
3 files changed, 86 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f80f65da/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
index cb848c0..306391b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
@@ -40,12 +40,13 @@ public class LlapStatusOptionsProcessor {
enum OptionConstants {
- NAME("name", 'n', "LLAP cluster name"),
+ NAME("name", 'n', "LLAP cluster name", true),
FIND_APP_TIMEOUT("findAppTimeout", 'f',
"Amount of time(s) that the tool will sleep to wait for the YARN application to start. negative values=wait forever, 0=Do not wait. default=" +
- TimeUnit.SECONDS.convert(FIND_YARN_APP_TIMEOUT_MS, TimeUnit.MILLISECONDS) + "s"),
+ TimeUnit.SECONDS.convert(FIND_YARN_APP_TIMEOUT_MS, TimeUnit.MILLISECONDS) + "s", true),
+ OUTPUT_FILE("outputFile", 'o', "File to which output should be written (Default stdout)", true),
HIVECONF("hiveconf", null, "Use value for given property. Overridden by explicit parameters", "property=value", 2),
- HELP("help", 'H', "Print help information");
+ HELP("help", 'H', "Print help information", false);
private final String longOpt;
@@ -54,9 +55,8 @@ public class LlapStatusOptionsProcessor {
private final String argName;
private final int numArgs;
- OptionConstants(String longOpt, char shortOpt, String description) {
- this(longOpt, shortOpt, description, longOpt, 1);
-
+ OptionConstants(String longOpt, char shortOpt, String description, boolean hasArgs) {
+ this(longOpt, shortOpt, description, longOpt, hasArgs ? 1 : 0);
}
OptionConstants(String longOpt, Character shortOpt, String description, String argName, int numArgs) {
@@ -93,11 +93,18 @@ public class LlapStatusOptionsProcessor {
private final String name;
private final Properties conf;
private final long findAppTimeoutMs;
+ private final String outputFile;
- LlapStatusOptions(String name, Properties hiveProperties, long findAppTimeoutMs) {
+ public LlapStatusOptions(String name, Properties hiveProperties, long findAppTimeoutMs,
+ String outputFile) {
this.name = name;
this.conf = hiveProperties;
this.findAppTimeoutMs = findAppTimeoutMs;
+ this.outputFile = outputFile;
+ }
+
+ public LlapStatusOptions(String name) {
+ this(name, new Properties(), FIND_YARN_APP_TIMEOUT_MS, null);
}
public String getName() {
@@ -111,6 +118,10 @@ public class LlapStatusOptionsProcessor {
public long getFindAppTimeoutMs() {
return findAppTimeoutMs;
}
+
+ public String getOutputFile() {
+ return outputFile;
+ }
}
private final Options options = new Options();
@@ -154,11 +165,16 @@ public class LlapStatusOptionsProcessor {
hiveConf = new Properties();
}
- return new LlapStatusOptions(name, hiveConf, findAppTimeoutMs);
+ String outputFile = null;
+ if (commandLine.hasOption(OptionConstants.OUTPUT_FILE.getLongOpt())) {
+ outputFile = commandLine.getOptionValue(OptionConstants.OUTPUT_FILE.getLongOpt());
+ }
+
+ return new LlapStatusOptions(name, hiveConf, findAppTimeoutMs, outputFile);
}
- private void printUsage() {
+ public static void printUsage() {
HelpFormatter hf = new HelpFormatter();
try {
int width = hf.getWidth();
@@ -167,7 +183,9 @@ public class LlapStatusOptionsProcessor {
hf.setWidth(width);
} catch (Throwable t) { // Ignore
}
- hf.printHelp(LLAPSTATUS_CONSTANT, options);
+
+ LlapStatusOptionsProcessor optionsProcessor = new LlapStatusOptionsProcessor();
+ hf.printHelp(LLAPSTATUS_CONSTANT, optionsProcessor.options);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f80f65da/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
index 646c286..5209226 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
@@ -19,7 +19,10 @@
package org.apache.hadoop.hive.llap.cli;
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.EnumSet;
import java.util.HashMap;
@@ -70,19 +73,29 @@ public class LlapStatusServiceDriver {
conf = (ss != null) ? ss.getConf() : new HiveConf(SessionState.class);
}
+ /**
+ * Parse command line options.
+ *
+ * @param args
+ * @return command line options.
+ */
+ public LlapStatusOptions parseOptions(String[] args) throws LlapStatusCliException {
+
+ LlapStatusOptionsProcessor optionsProcessor = new LlapStatusOptionsProcessor();
+ LlapStatusOptions options;
+ try {
+ options = optionsProcessor.processOptions(args);
+ return options;
+ } catch (Exception e) {
+ LOG.info("Failed to parse arguments", e);
+ throw new LlapStatusCliException(ExitCode.INCORRECT_USAGE, "Incorrect usage");
+ }
+ }
- public int run(String[] args) {
+ public int run(LlapStatusOptions options) {
SliderClient sliderClient = null;
try {
- LlapStatusOptionsProcessor optionsProcessor = new LlapStatusOptionsProcessor();
- LlapStatusOptions options;
- try {
- options = optionsProcessor.processOptions(args);
- } catch (Exception e) {
- LOG.info("Failed to parse arguments", e);
- return ExitCode.INCORRECT_USAGE.getInt();
- }
for (String f : LlapDaemonConfiguration.DAEMON_CONFIGS) {
conf.addResource(f);
@@ -111,6 +124,9 @@ public class LlapStatusServiceDriver {
LOG.info(message);
return ExitCode.INCORRECT_USAGE.getInt();
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using appName: {}", appName);
+ }
try {
sliderClient = createSliderClient();
@@ -157,7 +173,7 @@ public class LlapStatusServiceDriver {
return ret.getInt();
} else {
try {
- ret = populateAppStatusFromLlapRegistry(options, appStatusBuilder);
+ ret = populateAppStatusFromLlapRegistry(appName, appStatusBuilder);
} catch (LlapStatusCliException e) {
logError(e);
return e.getExitCode().getInt();
@@ -382,16 +398,16 @@ public class LlapStatusServiceDriver {
/**
*
- * @param options
+ * @param appName
* @param appStatusBuilder
* @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible
* @throws LlapStatusCliException
*/
- private ExitCode populateAppStatusFromLlapRegistry(LlapStatusOptions options, AppStatusBuilder appStatusBuilder) throws
+ private ExitCode populateAppStatusFromLlapRegistry(String appName, AppStatusBuilder appStatusBuilder) throws
LlapStatusCliException {
Configuration llapRegistryConf= new Configuration(conf);
llapRegistryConf
- .set(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + options.getName());
+ .set(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + appName);
LlapRegistryService llapRegistry;
try {
llapRegistry = LlapRegistryService.getClient(llapRegistryConf);
@@ -836,12 +852,35 @@ public class LlapStatusServiceDriver {
public static void main(String[] args) {
LOG.info("LLAP status invoked with arguments = {}", args);
- int ret;
+ int ret = ExitCode.SUCCESS.getInt();
+
+ LlapStatusServiceDriver statusServiceDriver = null;
+ LlapStatusOptions options = null;
+ try {
+ statusServiceDriver = new LlapStatusServiceDriver();
+ options = statusServiceDriver.parseOptions(args);
+ } catch (Throwable t) {
+ logError(t);
+ if (t instanceof LlapStatusCliException) {
+ LlapStatusCliException ce = (LlapStatusCliException) t;
+ ret = ce.getExitCode().getInt();
+ } else {
+ ret = ExitCode.INTERNAL_ERROR.getInt();
+ }
+ } finally {
+ LOG.info("LLAP status finished");
+ }
+ if (ret != 0 || options == null) { // Failure / help
+ System.exit(ret);
+ }
+
try {
- LlapStatusServiceDriver statusServiceDriver = new LlapStatusServiceDriver();
- ret = statusServiceDriver.run(args);
+ ret = statusServiceDriver.run(options);
if (ret == ExitCode.SUCCESS.getInt()) {
- try (PrintWriter pw = new PrintWriter(System.out)) {
+ try (OutputStream os = options.getOutputFile() == null ? System.out :
+ new BufferedOutputStream(
+ new FileOutputStream(options.getOutputFile())); PrintWriter pw = new PrintWriter(
+ os)) {
statusServiceDriver.outputJson(pw);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f80f65da/service/src/java/org/apache/hive/http/LlapServlet.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/http/LlapServlet.java b/service/src/java/org/apache/hive/http/LlapServlet.java
index 65e23fc..993766b 100644
--- a/service/src/java/org/apache/hive/http/LlapServlet.java
+++ b/service/src/java/org/apache/hive/http/LlapServlet.java
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.cli.LlapStatusOptionsProcessor;
import org.apache.hadoop.hive.llap.cli.LlapStatusServiceDriver;
@SuppressWarnings("serial")
@@ -97,7 +98,7 @@ public class LlapServlet extends HttpServlet {
LOG.info("Retrieving info for cluster: " + clusterName);
LlapStatusServiceDriver driver = new LlapStatusServiceDriver();
- int ret = driver.run(new String[] { "-n", clusterName });
+ int ret = driver.run(new LlapStatusOptionsProcessor.LlapStatusOptions(clusterName));
if (ret == LlapStatusServiceDriver.ExitCode.SUCCESS.getInt()) {
driver.outputJson(writer);
}