You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2013/11/04 16:26:31 UTC
svn commit: r1538648 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/processors/
ql/src/test/queries/clientnegative/ ql/src/test/queries/clientpositive/
ql/src/test/results/clientnegative/
Author: brock
Date: Mon Nov 4 15:26:31 2013
New Revision: 1538648
URL: http://svn.apache.org/r1538648
Log:
HIVE-5707: Validate values for ConfVar (Navis via Brock Noland)
Added:
hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q
hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
hive/trunk/ql/src/test/queries/clientpositive/orc_create.q
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1538648&r1=1538647&r2=1538648&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Nov 4 15:26:31 2013
@@ -27,10 +27,12 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -440,8 +442,10 @@ public class HiveConf extends Configurat
// Default file format for CREATE TABLE statement
// Options: TextFile, SequenceFile
- HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile"),
- HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile"),
+ HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile",
+ new StringsValidator("TextFile", "SequenceFile", "RCfile", "ORC")),
+ HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile",
+ new StringsValidator("TextFile", "SequenceFile", "RCfile")),
HIVECHECKFILEFORMAT("hive.fileformat.check", true),
// default serde for rcfile
@@ -654,7 +658,8 @@ public class HiveConf extends Configurat
HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true),
// 'minimal', 'more' (and 'all' later)
- HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal"),
+ HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal",
+ new StringsValidator("minimal", "more")),
HIVEFETCHTASKAGGR("hive.fetch.task.aggr", false),
@@ -732,7 +737,8 @@ public class HiveConf extends Configurat
HIVE_ENTITY_SEPARATOR("hive.entity.separator", "@"),
// binary or http
- HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary"),
+ HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary",
+ new StringsValidator("binary", "http")),
// http (over thrift) transport settings
HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001),
@@ -743,7 +749,8 @@ public class HiveConf extends Configurat
// binary transport settings
HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000),
HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", ""),
- HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth"),
+ HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth",
+ new StringsValidator("auth", "auth-int", "auth-conf")),
HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS("hive.server2.thrift.min.worker.threads", 5),
HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 500),
@@ -755,7 +762,8 @@ public class HiveConf extends Configurat
// HiveServer2 auth configuration
- HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", "NONE"),
+ HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", "NONE",
+ new StringsValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM")),
HIVE_SERVER2_KERBEROS_KEYTAB("hive.server2.authentication.kerberos.keytab", ""),
HIVE_SERVER2_KERBEROS_PRINCIPAL("hive.server2.authentication.kerberos.principal", ""),
HIVE_SERVER2_PLAIN_LDAP_URL("hive.server2.authentication.ldap.url", null),
@@ -763,7 +771,8 @@ public class HiveConf extends Configurat
HIVE_SERVER2_PLAIN_LDAP_DOMAIN("hive.server2.authentication.ldap.Domain", null),
HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class", null),
HIVE_SERVER2_ENABLE_DOAS("hive.server2.enable.doAs", true),
- HIVE_SERVER2_TABLE_TYPE_MAPPING("hive.server2.table.type.mapping", "CLASSIC"),
+ HIVE_SERVER2_TABLE_TYPE_MAPPING("hive.server2.table.type.mapping", "CLASSIC",
+ new StringsValidator("CLASSIC", "HIVE")),
HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", ""),
HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,delete,compile"),
@@ -824,7 +833,13 @@ public class HiveConf extends Configurat
private final VarType type;
+ private final Validator validator;
+
ConfVars(String varname, String defaultVal) {
+ this(varname, defaultVal, null);
+ }
+
+ ConfVars(String varname, String defaultVal, Validator validator) {
this.varname = varname;
this.valClass = String.class;
this.defaultVal = defaultVal;
@@ -833,9 +848,14 @@ public class HiveConf extends Configurat
this.defaultFloatVal = -1;
this.defaultBoolVal = false;
this.type = VarType.STRING;
+ this.validator = validator;
}
- ConfVars(String varname, int defaultIntVal) {
+ ConfVars(String varname, int defaultVal) {
+ this(varname, defaultVal, null);
+ }
+
+ ConfVars(String varname, int defaultIntVal, Validator validator) {
this.varname = varname;
this.valClass = Integer.class;
this.defaultVal = Integer.toString(defaultIntVal);
@@ -844,9 +864,14 @@ public class HiveConf extends Configurat
this.defaultFloatVal = -1;
this.defaultBoolVal = false;
this.type = VarType.INT;
+ this.validator = validator;
+ }
+
+ ConfVars(String varname, long defaultVal) {
+ this(varname, defaultVal, null);
}
- ConfVars(String varname, long defaultLongVal) {
+ ConfVars(String varname, long defaultLongVal, Validator validator) {
this.varname = varname;
this.valClass = Long.class;
this.defaultVal = Long.toString(defaultLongVal);
@@ -855,9 +880,14 @@ public class HiveConf extends Configurat
this.defaultFloatVal = -1;
this.defaultBoolVal = false;
this.type = VarType.LONG;
+ this.validator = validator;
}
- ConfVars(String varname, float defaultFloatVal) {
+ ConfVars(String varname, float defaultVal) {
+ this(varname, defaultVal, null);
+ }
+
+ ConfVars(String varname, float defaultFloatVal, Validator validator) {
this.varname = varname;
this.valClass = Float.class;
this.defaultVal = Float.toString(defaultFloatVal);
@@ -866,6 +896,7 @@ public class HiveConf extends Configurat
this.defaultFloatVal = defaultFloatVal;
this.defaultBoolVal = false;
this.type = VarType.FLOAT;
+ this.validator = validator;
}
ConfVars(String varname, boolean defaultBoolVal) {
@@ -877,12 +908,17 @@ public class HiveConf extends Configurat
this.defaultFloatVal = -1;
this.defaultBoolVal = defaultBoolVal;
this.type = VarType.BOOLEAN;
+ this.validator = null;
}
public boolean isType(String value) {
return type.isType(value);
}
+ public String validate(String value) {
+ return validator == null ? null : validator.validate(value);
+ }
+
public String typeString() {
return type.typeString();
}
@@ -1282,4 +1318,41 @@ public class HiveConf extends Configurat
}
}
+ /**
+ * validate value for a ConfVar, return non-null string for fail message
+ */
+ public static interface Validator {
+ String validate(String value);
+ }
+
+ public static class StringsValidator implements Validator {
+ private final Set<String> expected = new LinkedHashSet<String>();
+ private StringsValidator(String... values) {
+ for (String value : values) {
+ expected.add(value.toLowerCase());
+ }
+ }
+ @Override
+ public String validate(String value) {
+ if (value == null || !expected.contains(value.toLowerCase())) {
+ return "Invalid value.. expects one of " + expected;
+ }
+ return null;
+ }
+ }
+
+ public static class RatioValidator implements Validator {
+ @Override
+ public String validate(String value) {
+ try {
+ float fvalue = Float.valueOf(value);
+ if (fvalue <= 0 || fvalue >= 1) {
+ return "Invalid ratio " + value + ", which should be in between 0 to 1";
+ }
+ } catch (NumberFormatException e) {
+ return e.toString();
+ }
+ return null;
+ }
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1538648&r1=1538647&r2=1538648&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Mon Nov 4 15:26:31 2013
@@ -143,12 +143,21 @@ public class SetProcessor implements Com
String value = new VariableSubstitution().substitute(conf, varvalue);
if (conf.getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) {
HiveConf.ConfVars confVars = HiveConf.getConfVars(key);
- if (confVars != null && !confVars.isType(value)) {
- StringBuilder message = new StringBuilder();
- message.append("'SET ").append(varname).append('=').append(varvalue);
- message.append("' FAILED because "); message.append(key).append(" expects an ");
- message.append(confVars.typeString()).append(" value.");
- throw new IllegalArgumentException(message.toString());
+ if (confVars != null) {
+ if (!confVars.isType(value)) {
+ StringBuilder message = new StringBuilder();
+ message.append("'SET ").append(varname).append('=').append(varvalue);
+ message.append("' FAILED because ").append(key).append(" expects an ");
+ message.append(confVars.typeString()).append(" value.");
+ throw new IllegalArgumentException(message.toString());
+ }
+ String fail = confVars.validate(value);
+ if (fail != null) {
+ StringBuilder message = new StringBuilder();
+ message.append("'SET ").append(varname).append('=').append(varvalue);
+ message.append("' FAILED in validation : ").append(fail).append('.');
+ throw new IllegalArgumentException(message.toString());
+ }
}
}
conf.verifyAndSet(key, value);
Added: hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q?rev=1538648&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q Mon Nov 4 15:26:31 2013
@@ -0,0 +1,5 @@
+-- should fail: hive.fetch.task.conversion accepts minimal or more
+desc src;
+
+set hive.conf.validation=true;
+set hive.fetch.task.conversion=true;
Modified: hive/trunk/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_create.q?rev=1538648&r1=1538647&r2=1538648&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/orc_create.q Mon Nov 4 15:26:31 2013
@@ -40,7 +40,7 @@ set hive.default.fileformat=orc;
CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string);
-set hive.default.fileformat=text;
+set hive.default.fileformat=TextFile;
DESCRIBE FORMATTED orc_create;
Added: hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out?rev=1538648&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out Mon Nov 4 15:26:31 2013
@@ -0,0 +1,9 @@
+PREHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more
+desc src
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more
+desc src
+POSTHOOK: type: DESCTABLE
+key string default
+value string default
+Query returned non-zero code: 1, cause: 'SET hive.fetch.task.conversion=true' FAILED in validation : Invalid value.. expects one of [minimal, more].