You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/11/20 00:02:59 UTC
hive git commit: HIVE-12300 : deprecate MR in Hive 2.0 (Sergey
Shelukhin, reviewed by Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master d1c98a969 -> 6891128ca
HIVE-12300 : deprecate MR in Hive 2.0 (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6891128c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6891128c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6891128c
Branch: refs/heads/master
Commit: 6891128caabc04c084582f34a948e7c4cad1ebd6
Parents: d1c98a9
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Nov 19 15:02:29 2015 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Nov 19 15:02:29 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/hive/beeline/BeeLine.java | 13 ++++++-
.../org/apache/hadoop/hive/cli/CliDriver.java | 8 ++++
.../org/apache/hadoop/hive/conf/HiveConf.java | 29 ++++++++++++++-
.../java/org/apache/hadoop/hive/ql/Driver.java | 20 +++++++++-
.../ql/processors/CommandProcessorResponse.java | 15 ++++++--
.../hadoop/hive/ql/processors/SetProcessor.java | 39 ++++++++++++++------
.../hadoop/hive/ql/session/OperationLog.java | 10 +++++
.../cli/operation/HiveCommandOperation.java | 11 ++++++
8 files changed, 127 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 377703f..1e289ca 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -109,6 +109,7 @@ import org.apache.hive.beeline.cli.CliOptionsProcessor;
* </ul>
*
*/
+@SuppressWarnings("static-access")
public class BeeLine implements Closeable {
private static final ResourceBundle resourceBundle =
ResourceBundle.getBundle(BeeLine.class.getSimpleName());
@@ -657,7 +658,7 @@ public class BeeLine implements Closeable {
Properties confProps = commandLine.getOptionProperties("hiveconf");
for (String propKey : confProps.stringPropertyNames()) {
- getOpts().getHiveConfVariables().put(propKey, confProps.getProperty(propKey));
+ setHiveConfVar(propKey, confProps.getProperty(propKey));
}
Properties hiveVars = commandLine.getOptionProperties("define");
@@ -739,7 +740,7 @@ public class BeeLine implements Closeable {
Properties hiveConfs = cl.getOptionProperties("hiveconf");
for (String key : hiveConfs.stringPropertyNames()) {
- getOpts().getHiveConfVariables().put(key, hiveConfs.getProperty(key));
+ setHiveConfVar(key, hiveConfs.getProperty(key));
}
driver = cl.getOptionValue("d");
@@ -794,6 +795,14 @@ public class BeeLine implements Closeable {
return code;
}
+
+ private void setHiveConfVar(String key, String val) {
+ getOpts().getHiveConfVariables().put(key, val);
+ if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) {
+ info(HiveConf.generateMrDeprecationWarning());
+ }
+ }
+
private String constructCmd(String url, String user, String pass, String driver, boolean stripPasswd) {
String com = "!connect "
+ url + " "
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
----------------------------------------------------------------------
diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
index b359850..e04f247 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
@@ -290,6 +290,11 @@ public class CliDriver {
ss.out.println("Query returned non-zero code: " + res.getResponseCode() +
", cause: " + res.getErrorMessage());
}
+ if (res.getConsoleMessages() != null) {
+ for (String consoleMsg : res.getConsoleMessages()) {
+ console.printInfo(consoleMsg);
+ }
+ }
ret = res.getResponseCode();
}
}
@@ -748,6 +753,9 @@ public class CliDriver {
System.err.println("Could not open input file for reading. (" + e.getMessage() + ")");
return 3;
}
+ if ("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE))) {
+ console.printInfo(HiveConf.generateMrDeprecationWarning());
+ }
setupConsoleReader();
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2185f85..0afb964 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2164,7 +2164,9 @@ public class HiveConf extends Configuration {
"Whether to show the unquoted partition names in query results."),
HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr", new StringSet("mr", "tez", "spark"),
- "Chooses execution engine. Options are: mr (Map reduce, default), tez (hadoop 2 only), spark"),
+ "Chooses execution engine. Options are: mr (Map reduce, default), tez, spark. While MR\n" +
+ "remains the default engine for historical reasons, it is itself a historical engine\n" +
+ "and is deprecated in Hive 2 line. It may be removed without further warning."),
HIVE_EXECUTION_MODE("hive.execution.mode", "container", new StringSet("container", "llap"),
"Chooses whether query fragments will run in container or in llap"),
@@ -2571,6 +2573,13 @@ public class HiveConf extends Configuration {
return defaultExpr;
}
+ private Set<String> getValidStringValues() {
+ if (validator == null || !(validator instanceof StringSet)) {
+ throw new RuntimeException(varname + " does not specify a list of valid values");
+ }
+ return ((StringSet)validator).getExpected();
+ }
+
enum VarType {
STRING {
@Override
@@ -3394,4 +3403,22 @@ public class HiveConf extends Configuration {
public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) {
HiveConf.loadHiveServer2Config = loadHiveServer2Config;
}
+
+ public static String getNonMrEngines() {
+ String result = "";
+ for (String s : ConfVars.HIVE_EXECUTION_ENGINE.getValidStringValues()) {
+ if ("mr".equals(s)) continue;
+ if (!result.isEmpty()) {
+ result += ", ";
+ }
+ result += s;
+ }
+ return result;
+ }
+
+ public static String generateMrDeprecationWarning() {
+ return "Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. "
+ + "Consider using a different execution engine (i.e. " + HiveConf.getNonMrEngines()
+ + ") or using Hive 1.X releases.";
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index fc5a951..fbde9eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -114,6 +114,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObje
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivObjectActionType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.ByteStream;
@@ -1505,10 +1506,12 @@ public class Driver implements CommandProcessor {
}
}
- int jobs = Utilities.getMRTasks(plan.getRootTasks()).size()
+ int mrJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
+ int jobs = mrJobs
+ Utilities.getTezTasks(plan.getRootTasks()).size()
+ Utilities.getSparkTasks(plan.getRootTasks()).size();
if (jobs > 0) {
+ logMrWarning(mrJobs);
console.printInfo("Query ID = " + plan.getQueryId());
console.printInfo("Total jobs = " + jobs);
}
@@ -1730,6 +1733,21 @@ public class Driver implements CommandProcessor {
return (0);
}
+ private void logMrWarning(int mrJobs) {
+ if (mrJobs <= 0 || !("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE)))) {
+ return;
+ }
+ String warning = HiveConf.generateMrDeprecationWarning();
+ LOG.warn(warning);
+ warning = "WARNING: " + warning;
+ console.printInfo(warning);
+ // Propagate warning to beeline via operation log.
+ OperationLog ol = OperationLog.getCurrentOperationLog();
+ if (ol != null) {
+ ol.writeOperationLog(LoggingLevel.EXECUTION, warning + "\n");
+ }
+ }
+
private void setErrorMsgAndDetail(int exitVal, Throwable downstreamError, Task tsk) {
this.downstreamError = downstreamError;
errorMessage = "FAILED: Execution Error, return code " + exitVal + " from " + tsk.getClass().getName();
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
index 21b7457..d4f35f1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.processors;
+import java.util.List;
+
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -36,6 +38,7 @@ public class CommandProcessorResponse {
private final Schema resSchema;
private final Throwable exception;
+ private final List<String> consoleMessages;
public CommandProcessorResponse(int responseCode) {
this(responseCode, null, null, null, null);
@@ -45,6 +48,10 @@ public class CommandProcessorResponse {
this(responseCode, errorMessage, SQLState, null, null);
}
+ public CommandProcessorResponse(int responseCode, List<String> consoleMessages) {
+ this(responseCode, null, null, null, null, -1, consoleMessages);
+ }
+
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, Throwable exception) {
this(responseCode, errorMessage, SQLState, null, exception);
}
@@ -54,7 +61,7 @@ public class CommandProcessorResponse {
}
public CommandProcessorResponse(int responseCode, ErrorMsg canonicalErrMsg, Throwable t, String ... msgArgs) {
this(responseCode, canonicalErrMsg.format(msgArgs),
- canonicalErrMsg.getSQLState(), null, t, canonicalErrMsg.getErrorCode());
+ canonicalErrMsg.getSQLState(), null, t, canonicalErrMsg.getErrorCode(), null);
}
/**
@@ -71,16 +78,17 @@ public class CommandProcessorResponse {
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState,
Schema schema, Throwable exception) {
- this(responseCode, errorMessage, SQLState, schema, exception, -1);
+ this(responseCode, errorMessage, SQLState, schema, exception, -1, null);
}
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState,
- Schema schema, Throwable exception, int hiveErrorCode) {
+ Schema schema, Throwable exception, int hiveErrorCode, List<String> consoleMessages) {
this.responseCode = responseCode;
this.errorMessage = errorMessage;
this.SQLState = SQLState;
this.resSchema = schema;
this.exception = exception;
this.hiveErrorCode = hiveErrorCode;
+ this.consoleMessages = consoleMessages;
}
public int getResponseCode() { return responseCode; }
@@ -89,6 +97,7 @@ public class CommandProcessorResponse {
public Schema getSchema() { return resSchema; }
public Throwable getException() { return exception; }
public int getErrorCode() { return hiveErrorCode; }
+ public List<String> getConsoleMessages() { return consoleMessages; }
public String toString() {
return "(" + responseCode + "," + errorMessage + "," +
(hiveErrorCode > 0 ? hiveErrorCode + "," : "" ) +
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index 9389759..9a3ba04 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -37,12 +37,17 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
/**
* SetProcessor.
*
*/
public class SetProcessor implements CommandProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(SetProcessor.class);
private static final String prefix = "set: ";
@@ -110,22 +115,24 @@ public class SetProcessor implements CommandProcessor {
public CommandProcessorResponse executeSetVariable(String varname, String varvalue) {
try {
- return new CommandProcessorResponse(setVariable(varname, varvalue));
+ return setVariable(varname, varvalue);
} catch (Exception e) {
return new CommandProcessorResponse(1, e.getMessage(), "42000",
e instanceof IllegalArgumentException ? null : e);
}
}
- public static int setVariable(String varname, String varvalue) throws Exception {
+ public static CommandProcessorResponse setVariable(
+ String varname, String varvalue) throws Exception {
SessionState ss = SessionState.get();
if (varvalue.contains("\n")){
ss.err.println("Warning: Value had a \\n character in it.");
}
varname = varname.trim();
+ String nonErrorMessage = null;
if (varname.startsWith(ENV_PREFIX)){
ss.err.println("env:* variables can not be set.");
- return 1;
+ return new CommandProcessorResponse(1); // Should we propagate the error message properly?
} else if (varname.startsWith(SYSTEM_PREFIX)){
String propName = varname.substring(SYSTEM_PREFIX.length());
System.getProperties()
@@ -137,7 +144,7 @@ public class SetProcessor implements CommandProcessor {
}).substitute(ss.getConf(), varvalue));
} else if (varname.startsWith(HIVECONF_PREFIX)){
String propName = varname.substring(HIVECONF_PREFIX.length());
- setConf(varname, propName, varvalue, false);
+ nonErrorMessage = setConf(varname, propName, varvalue, false);
} else if (varname.startsWith(HIVEVAR_PREFIX)) {
String propName = varname.substring(HIVEVAR_PREFIX.length());
ss.getHiveVariables().put(propName, new VariableSubstitution(new HiveVariableSource() {
@@ -156,17 +163,21 @@ public class SetProcessor implements CommandProcessor {
}
}).substitute(ss.getConf(), varvalue));
} else {
- setConf(varname, varname, varvalue, true);
+ nonErrorMessage = setConf(varname, varname, varvalue, true);
if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) {
SessionState.get().updateHistory(Boolean.parseBoolean(varvalue), ss);
}
}
- return 0;
+ return nonErrorMessage == null ? new CommandProcessorResponse(0)
+ : new CommandProcessorResponse(0, Lists.newArrayList(nonErrorMessage));
}
- // returns non-null string for validation fail
- private static void setConf(String varname, String key, String varvalue, boolean register)
+ /**
+ * @return A console message that is not strong enough to fail the command (e.g. deprecation).
+ */
+ private static String setConf(String varname, String key, String varvalue, boolean register)
throws IllegalArgumentException {
+ String result = null;
HiveConf conf = SessionState.get().getConf();
String value = new VariableSubstitution(new HiveVariableSource() {
@Override
@@ -196,13 +207,19 @@ public class SetProcessor implements CommandProcessor {
}
}
conf.verifyAndSet(key, value);
- if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)
- && !"spark".equals(value)) {
- SessionState.get().closeSparkSession();
+ if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)) {
+ if (!"spark".equals(value)) {
+ SessionState.get().closeSparkSession();
+ }
+ if ("mr".equals(value)) {
+ result = HiveConf.generateMrDeprecationWarning();
+ LOG.warn(result);
+ }
}
if (register) {
SessionState.get().getOverriddenConfigurations().put(key, value);
}
+ return result;
}
private SortedMap<String,String> propertiesToSortedMap(Properties p){
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
index fb7dda5..2ecdde9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
@@ -103,6 +103,16 @@ public class OperationLog {
}
/**
+ * Write operation execution logs into log file
+ * @param operationLogMessage one line of log emitted from log4j
+ */
+ public void writeOperationLog(LoggingLevel level, String operationLogMessage) {
+ if (opLoggingLevel.compareTo(level) < 0) return;
+ logFile.write(operationLogMessage);
+ }
+
+
+ /**
* Read operation execution logs from log file
* @param isFetchFirst true if the Enum FetchOrientation value is Fetch_First
* @param maxRows the max number of fetched lines from log
http://git-wip-us.apache.org/repos/asf/hive/blob/6891128c/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
index 22c55f1..04dc6e3 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
@@ -34,7 +34,9 @@ import org.apache.commons.lang3.CharEncoding;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.OperationLog;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
import org.apache.hive.service.ServiceUtils;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
@@ -123,6 +125,15 @@ public class HiveCommandOperation extends ExecuteStatementOperation {
setHasResultSet(false);
resultSchema = new TableSchema();
}
+ if (response.getConsoleMessages() != null) {
+ // Propagate processor messages (if any) to beeline or other client.
+ OperationLog ol = OperationLog.getCurrentOperationLog();
+ if (ol != null) {
+ for (String consoleMsg : response.getConsoleMessages()) {
+ ol.writeOperationLog(LoggingLevel.EXECUTION, consoleMsg + "\n");
+ }
+ }
+ }
} catch (HiveSQLException e) {
setState(OperationState.ERROR);
throw e;