You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/08/30 22:21:12 UTC
[05/17] hive git commit: HIVE-14418 : Hive config validation prevents
unsetting the settings (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
HIVE-14418 : Hive config validation prevents unsetting the settings (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0705323d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0705323d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0705323d
Branch: refs/heads/hive-14535
Commit: 0705323db28edf13777d29d3a0add48f19936db0
Parents: b80bcd0
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Aug 29 15:36:39 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Aug 29 15:36:39 2016 -0700
----------------------------------------------------------------------
.../hive/ql/processors/ResetProcessor.java | 109 +++++++++++++++++--
.../hadoop/hive/ql/processors/SetProcessor.java | 3 +-
ql/src/test/queries/clientpositive/reset_conf.q | 18 +++
.../test/queries/clientpositive/set_metaconf.q | 4 +
.../results/clientpositive/reset_conf.q.out | 32 ++++++
.../results/clientpositive/set_metaconf.q.out | 2 +
6 files changed, 158 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
index e67422b..bbd4501 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
@@ -18,41 +18,134 @@
package org.apache.hadoop.hive.ql.processors;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.SystemVariables;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import org.apache.hadoop.hive.ql.session.SessionState;
+import com.google.common.collect.Lists;
+
public class ResetProcessor implements CommandProcessor {
@Override
public void init() {
}
+ private final static String DEFAULT_ARG = "-d";
+
@Override
public CommandProcessorResponse run(String command) throws CommandNeedRetryException {
SessionState ss = SessionState.get();
CommandProcessorResponse authErrResp =
CommandUtil.authorizeCommand(ss, HiveOperationType.RESET, Arrays.asList(command));
- if(authErrResp != null){
+ if (authErrResp != null) {
// there was an authorization issue
return authErrResp;
}
-
- if (ss.getOverriddenConfigurations().isEmpty()) {
+ command = command.trim();
+ if (StringUtils.isBlank(command)) {
+ resetOverridesOnly(ss);
return new CommandProcessorResponse(0);
}
+ String[] parts = command.split("\\s+");
+ boolean isDefault = false;
+ List<String> varnames = new ArrayList<>(parts.length);
+ for (String part : parts) {
+ if (part.isEmpty()) continue;
+ if (DEFAULT_ARG.equals(part)) {
+ isDefault = true;
+ } else {
+ varnames.add(part);
+ }
+ }
+ if (varnames.isEmpty()) {
+ return new CommandProcessorResponse(1, "No variable names specified", "42000");
+ }
+ String message = "";
+ for (String varname : varnames) {
+ if (isDefault) {
+ if (!message.isEmpty()) {
+ message += ", ";
+ }
+ message += varname;
+ resetToDefault(ss, varname);
+ } else {
+ resetOverrideOnly(ss, varname);
+ }
+ }
+ return new CommandProcessorResponse(0, isDefault
+ ? Lists.newArrayList("Resetting " + message + " to default values") : null);
+ }
+
+ private void resetOverridesOnly(SessionState ss) {
+ if (ss.getOverriddenConfigurations().isEmpty()) return;
HiveConf conf = new HiveConf();
for (String key : ss.getOverriddenConfigurations().keySet()) {
- String value = conf.get(key);
- if (value != null) {
- ss.getConf().set(key, value);
- }
+ setSessionVariableFromConf(ss, key, conf);
}
ss.getOverriddenConfigurations().clear();
- return new CommandProcessorResponse(0);
+ }
+
+ private void resetOverrideOnly(SessionState ss, String varname) {
+ if (!ss.getOverriddenConfigurations().containsKey(varname)) return;
+ setSessionVariableFromConf(ss, varname, new HiveConf());
+ ss.getOverriddenConfigurations().remove(varname);
+ }
+
+ private void setSessionVariableFromConf(SessionState ss, String varname,
+ HiveConf conf) {
+ String value = conf.get(varname);
+ if (value != null) {
+ ss.getConf().set(varname, value);
+ }
+ }
+
+ private CommandProcessorResponse resetToDefault(SessionState ss, String varname) {
+ varname = varname.trim();
+ try {
+ String nonErrorMessage = null;
+ if (varname.startsWith(SystemVariables.HIVECONF_PREFIX)){
+ String propName = varname.substring(SystemVariables.HIVECONF_PREFIX.length());
+ nonErrorMessage = SetProcessor.setConf(
+ varname, propName, getConfVar(propName).getDefaultValue(), false);
+ } else if (varname.startsWith(SystemVariables.METACONF_PREFIX)) {
+ String propName = varname.substring(SystemVariables.METACONF_PREFIX.length());
+ HiveConf.ConfVars confVars = getConfVar(propName);
+ Hive.get(ss.getConf()).setMetaConf(propName, new VariableSubstitution(new HiveVariableSource() {
+ @Override
+ public Map<String, String> getHiveVariable() {
+ return SessionState.get().getHiveVariables();
+ }
+ }).substitute(ss.getConf(), confVars.getDefaultValue()));
+ } else {
+ String defaultVal = getConfVar(varname).getDefaultValue();
+ nonErrorMessage = SetProcessor.setConf(varname, varname, defaultVal, true);
+ if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) {
+ SessionState.get().updateHistory(Boolean.parseBoolean(defaultVal), ss);
+ }
+ }
+ return nonErrorMessage == null ? new CommandProcessorResponse(0)
+ : new CommandProcessorResponse(0, Lists.newArrayList(nonErrorMessage));
+ } catch (Exception e) {
+ return new CommandProcessorResponse(1, e.getMessage(), "42000",
+ e instanceof IllegalArgumentException ? null : e);
+ }
+ }
+
+ private static HiveConf.ConfVars getConfVar(String propName) {
+ HiveConf.ConfVars confVars = HiveConf.getConfVars(propName);
+ if (confVars == null) throw new IllegalArgumentException(propName + " not found");
+ return confVars;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index 2e13dab..c9d06ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -24,7 +24,6 @@ import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultN
import static org.apache.hadoop.hive.conf.SystemVariables.*;
-import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@@ -179,7 +178,7 @@ public class SetProcessor implements CommandProcessor {
/**
* @return A console message that is not strong enough to fail the command (e.g. deprecation).
*/
- private static String setConf(String varname, String key, String varvalue, boolean register)
+ static String setConf(String varname, String key, String varvalue, boolean register)
throws IllegalArgumentException {
String result = null;
HiveConf conf = SessionState.get().getConf();
http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/queries/clientpositive/reset_conf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/reset_conf.q b/ql/src/test/queries/clientpositive/reset_conf.q
index 8ddde23..8420d02 100644
--- a/ql/src/test/queries/clientpositive/reset_conf.q
+++ b/ql/src/test/queries/clientpositive/reset_conf.q
@@ -9,3 +9,21 @@ reset;
set hive.skewjoin.key;
set hive.skewjoin.mapjoin.min.split;
+
+set hive.skewjoin.key=300000;
+set hive.skewjoin.mapjoin.min.split=256000000;
+select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split';
+set hive.skewjoin.key;
+
+reset -d hive.skewjoin.key;
+select 'After resetting hive.skewjoin.key to default';
+set hive.skewjoin.key;
+set hive.skewjoin.mapjoin.min.split;
+
+set hive.skewjoin.key=300000;
+
+reset -d hive.skewjoin.key hive.skewjoin.mapjoin.min.split;
+select 'After resetting both to default';
+set hive.skewjoin.key;
+set hive.skewjoin.mapjoin.min.split;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/queries/clientpositive/set_metaconf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/set_metaconf.q b/ql/src/test/queries/clientpositive/set_metaconf.q
index a679489..6186122 100644
--- a/ql/src/test/queries/clientpositive/set_metaconf.q
+++ b/ql/src/test/queries/clientpositive/set_metaconf.q
@@ -4,3 +4,7 @@ set metaconf:hive.metastore.try.direct.sql;
set metaconf:hive.metastore.try.direct.sql=false;
set metaconf:hive.metastore.try.direct.sql;
set hive.metastore.try.direct.sql;
+
+reset -d metaconf:hive.metastore.try.direct.sql;
+set metaconf:hive.metastore.try.direct.sql;
+set hive.metastore.try.direct.sql;
http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/results/clientpositive/reset_conf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/reset_conf.q.out b/ql/src/test/results/clientpositive/reset_conf.q.out
index e4e15be..12f2555 100644
--- a/ql/src/test/results/clientpositive/reset_conf.q.out
+++ b/ql/src/test/results/clientpositive/reset_conf.q.out
@@ -4,3 +4,35 @@ hive.skewjoin.key=300000
hive.skewjoin.mapjoin.min.split=256000000
hive.skewjoin.key=100000
hive.skewjoin.mapjoin.min.split=33554432
+PREHOOK: query: select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split
+hive.skewjoin.key=300000
+PREHOOK: query: select 'After resetting hive.skewjoin.key to default'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'After resetting hive.skewjoin.key to default'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+After resetting hive.skewjoin.key to default
+hive.skewjoin.key=100000
+hive.skewjoin.mapjoin.min.split=256000000
+PREHOOK: query: select 'After resetting both to default'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'After resetting both to default'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+After resetting both to default
+hive.skewjoin.key=100000
+hive.skewjoin.mapjoin.min.split=33554432
http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/results/clientpositive/set_metaconf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/set_metaconf.q.out b/ql/src/test/results/clientpositive/set_metaconf.q.out
index ec33e59..41b8957 100644
--- a/ql/src/test/results/clientpositive/set_metaconf.q.out
+++ b/ql/src/test/results/clientpositive/set_metaconf.q.out
@@ -1,3 +1,5 @@
metaconf:hive.metastore.try.direct.sql=true
metaconf:hive.metastore.try.direct.sql=false
hive.metastore.try.direct.sql=true
+metaconf:hive.metastore.try.direct.sql=true
+hive.metastore.try.direct.sql=true