You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2011/09/02 03:04:58 UTC
svn commit: r1164319 - in /hive/trunk:
cli/src/java/org/apache/hadoop/hive/cli/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/java/org/apache/hadoop/hive/ql/processors/
ql/src/java/org/apache/hadoop/hive/ql/session/
ql/src/test/queries/clientposi...
Author: cws
Date: Fri Sep 2 01:04:58 2011
New Revision: 1164319
URL: http://svn.apache.org/viewvc?rev=1164319&view=rev
Log:
HIVE-2020. Create a separate namespace for Hive variables (Vaibhav Aggarwal via cws)
Added:
hive/trunk/ql/src/test/queries/clientpositive/set_variable_sub.q
hive/trunk/ql/src/test/results/clientpositive/set_variable_sub.q.out
Modified:
hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
Modified: hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=1164319&r1=1164318&r2=1164319&view=diff
==============================================================================
--- hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Fri Sep 2 01:04:58 2011
@@ -621,6 +621,7 @@ public class CliDriver {
}
CliDriver cli = new CliDriver();
+ cli.setHiveVariables(oproc.getHiveVariables());
// Execute -i init files (always in silent mode)
cli.processInitFiles(ss);
@@ -710,4 +711,8 @@ public class CliDriver {
return String.format("%1$-" + s.length() +"s", "");
}
+ public void setHiveVariables(Map<String, String> hiveVariables) {
+ SessionState.get().setHiveVariables(hiveVariables);
+ }
+
}
Modified: hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java?rev=1164319&r1=1164318&r2=1164319&view=diff
==============================================================================
--- hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java (original)
+++ hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java Fri Sep 2 01:04:58 2011
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.cli;
+import java.util.HashMap;
import java.util.Arrays;
+import java.util.Map;
import java.util.Properties;
import org.apache.commons.cli.GnuParser;
@@ -38,7 +40,7 @@ public class OptionsProcessor {
protected static final Log l4j = LogFactory.getLog(OptionsProcessor.class.getName());
private final Options options = new Options();
private org.apache.commons.cli.CommandLine commandLine;
-
+ Map<String, String> hiveVariables = new HashMap<String, String>();
@SuppressWarnings("static-access")
public OptionsProcessor() {
@@ -87,6 +89,24 @@ public class OptionsProcessor {
.withDescription("connecting to Hive Server on port number")
.create('p'));
+ // Substitution option -d, --define
+ options.addOption(OptionBuilder
+ .withValueSeparator()
+ .hasArgs(2)
+ .withArgName("key=value")
+ .withLongOpt("define")
+ .withDescription("Variable subsitution to apply to hive commands. e.g. -d A=B or --define A=B")
+ .create('d'));
+
+ // Substitution option --hivevar
+ options.addOption(OptionBuilder
+ .withValueSeparator()
+ .hasArgs(2)
+ .withArgName("key=value")
+ .withLongOpt("hivevar")
+ .withDescription("Variable subsitution to apply to hive commands. e.g. --hivevar A=B")
+ .create());
+
// [-S|--silent]
options.addOption(new Option("S", "silent", false, "Silent mode in interactive shell"));
@@ -105,6 +125,16 @@ public class OptionsProcessor {
for (String propKey : confProps.stringPropertyNames()) {
System.setProperty(propKey, confProps.getProperty(propKey));
}
+
+ Properties hiveVars = commandLine.getOptionProperties("define");
+ for (String propKey : hiveVars.stringPropertyNames()) {
+ hiveVariables.put(propKey, hiveVars.getProperty(propKey));
+ }
+
+ Properties hiveVars2 = commandLine.getOptionProperties("hivevar");
+ for (String propKey : hiveVars2.stringPropertyNames()) {
+ hiveVariables.put(propKey, hiveVars2.getProperty(propKey));
+ }
} catch (ParseException e) {
System.err.println(e.getMessage());
printUsage();
@@ -157,4 +187,8 @@ public class OptionsProcessor {
private void printUsage() {
new HelpFormatter().printHelp("hive", options);
}
+
+ public Map<String, String> getHiveVariables() {
+ return hiveVariables;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java?rev=1164319&r1=1164318&r2=1164319&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java Fri Sep 2 01:04:58 2011
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.processors.SetProcessor;
+import org.apache.hadoop.hive.ql.session.SessionState;
public class VariableSubstitution {
@@ -32,6 +33,35 @@ public class VariableSubstitution {
protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
protected static int MAX_SUBST = 40;
+ private String getSubstitute(HiveConf conf, String var) {
+ String val = null;
+ try {
+ if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) {
+ val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length()));
+ }
+ } catch(SecurityException se) {
+ l4j.warn("Unexpected SecurityException in Configuration", se);
+ }
+ if (val ==null){
+ if (var.startsWith(SetProcessor.ENV_PREFIX)){
+ val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length()));
+ }
+ }
+ if (val == null) {
+ if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){
+ val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length()));
+ }
+ }
+ if (val ==null){
+ if(var.startsWith(SetProcessor.HIVEVAR_PREFIX)){
+ val = SessionState.get().getHiveVariables().get(var.substring(SetProcessor.HIVEVAR_PREFIX.length()));
+ } else {
+ val = SessionState.get().getHiveVariables().get(var);
+ }
+ }
+ return val;
+ }
+
public String substitute (HiveConf conf, String expr) {
if (conf.getBoolVar(ConfVars.HIVEVARIABLESUBSTITUTE)){
@@ -51,27 +81,11 @@ public class VariableSubstitution {
}
String var = match.group();
var = var.substring(2, var.length()-1); // remove ${ .. }
- String val = null;
- try {
- if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) {
- val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length()));
- }
- } catch(SecurityException se) {
- l4j.warn("Unexpected SecurityException in Configuration", se);
- }
- if (val ==null){
- if (var.startsWith(SetProcessor.ENV_PREFIX)){
- val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length()));
- }
- }
- if (val == null) {
- if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){
- val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length()));
- }
- }
+ String val = getSubstitute(conf, var);
+
if (val == null) {
l4j.debug("Interpolation result: "+eval);
- return eval; // return literal ${var}: var is unbound
+ return eval; // return literal, no substitution found
}
// substitute
eval = eval.substring(0, match.start())+val+eval.substring(match.end());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1164319&r1=1164318&r2=1164319&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Fri Sep 2 01:04:58 2011
@@ -36,6 +36,7 @@ public class SetProcessor implements Com
public static final String ENV_PREFIX = "env:";
public static final String SYSTEM_PREFIX = "system:";
public static final String HIVECONF_PREFIX = "hiveconf:";
+ public static final String HIVEVAR_PREFIX = "hivevar:";
public static boolean getBoolean(String value) {
if (value.equals("on") || value.equals("true")) {
@@ -50,7 +51,6 @@ public class SetProcessor implements Com
private void dumpOptions(Properties p) {
SessionState ss = SessionState.get();
-
SortedMap<String, String> sortedMap = new TreeMap<String, String>();
sortedMap.put("silent", (ss.getIsSilent() ? "on" : "off"));
for (Object one : p.keySet()) {
@@ -58,6 +58,12 @@ public class SetProcessor implements Com
String oneValue = p.getProperty(oneProp);
sortedMap.put(oneProp, oneValue);
}
+
+ // Inserting hive variables
+ for (String s : ss.getHiveVariables().keySet()) {
+ sortedMap.put(SetProcessor.HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s));
+ }
+
for (Map.Entry<String, String> entries : sortedMap.entrySet()) {
ss.out.println(entries.getKey() + "=" + entries.getValue());
}
@@ -73,11 +79,13 @@ public class SetProcessor implements Com
}
- private void dumpOption(Properties p, String s) {
+ private void dumpOption(String s) {
SessionState ss = SessionState.get();
- if (p.getProperty(s) != null) {
- ss.out.println(s + "=" + p.getProperty(s));
+ if (ss.getConf().get(s) != null) {
+ ss.out.println(s + "=" + ss.getConf().get(s));
+ } else if (ss.getHiveVariables().containsKey(s)) {
+ ss.out.println(s + "=" + ss.getHiveVariables().get(s));
} else {
ss.out.println(s + " is undefined");
}
@@ -99,6 +107,10 @@ public class SetProcessor implements Com
String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length());
ss.getConf().set(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
return new CommandProcessorResponse(0);
+ } else if (varname.startsWith(SetProcessor.HIVEVAR_PREFIX)) {
+ String propName = varname.substring(SetProcessor.HIVEVAR_PREFIX.length());
+ ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
+ return new CommandProcessorResponse(0);
} else {
ss.getConf().set(varname, new VariableSubstitution().substitute(ss.getConf(),varvalue) );
return new CommandProcessorResponse(0);
@@ -154,8 +166,17 @@ public class SetProcessor implements Com
ss.out.println(varname + " is undefined as a hive configuration variable");
return new CommandProcessorResponse(1);
}
+ } else if (varname.indexOf(SetProcessor.HIVEVAR_PREFIX)==0) {
+ String var = varname.substring(SetProcessor.HIVEVAR_PREFIX.length());
+ if (ss.getHiveVariables().get(var)!=null){
+ ss.out.println(SetProcessor.HIVEVAR_PREFIX+var + "=" + ss.getHiveVariables().get(var));
+ return new CommandProcessorResponse(0);
+ } else {
+ ss.out.println(varname + " is undefined as a hive variable");
+ return new CommandProcessorResponse(1);
+ }
} else {
- dumpOption(ss.getConf().getAllProperties(), varname);
+ dumpOption(varname);
return new CommandProcessorResponse(0);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1164319&r1=1164318&r2=1164319&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri Sep 2 01:04:58 2011
@@ -28,6 +28,7 @@ import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -110,6 +111,8 @@ public class SessionState {
private List<MapRedStats> lastMapRedStatsList;
+ private Map<String, String> hiveVariables;
+
/**
* Lineage state.
*/
@@ -185,6 +188,17 @@ public class SessionState {
return (conf.getVar(HiveConf.ConfVars.HIVEQUERYID));
}
+ public Map<String, String> getHiveVariables() {
+ if (hiveVariables == null) {
+ hiveVariables = new HashMap<String, String>();
+ }
+ return hiveVariables;
+ }
+
+ public void setHiveVariables(Map<String, String> hiveVariables) {
+ this.hiveVariables = hiveVariables;
+ }
+
public String getSessionId() {
return (conf.getVar(HiveConf.ConfVars.HIVESESSIONID));
}
Added: hive/trunk/ql/src/test/queries/clientpositive/set_variable_sub.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/set_variable_sub.q?rev=1164319&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/set_variable_sub.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/set_variable_sub.q Fri Sep 2 01:04:58 2011
@@ -0,0 +1,15 @@
+set hivevar:key1=value1;
+
+EXPLAIN SELECT * FROM src where key="${key1}";
+EXPLAIN SELECT * FROM src where key="${hivevar:key1}";
+
+set hivevar:a=1;
+set hivevar:b=a;
+set hivevar:c=${hivevar:${hivevar:b}};
+EXPLAIN SELECT * FROM src where key="${hivevar:c}";
+
+set hivevar:a;
+set hivevar:b;
+set hivevar:c;
+set hivevar:key1;
+
Added: hive/trunk/ql/src/test/results/clientpositive/set_variable_sub.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/set_variable_sub.q.out?rev=1164319&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/set_variable_sub.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/set_variable_sub.q.out Fri Sep 2 01:04:58 2011
@@ -0,0 +1,127 @@
+PREHOOK: query: EXPLAIN SELECT * FROM src where key="value1"
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT * FROM src where key="value1"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) "value1"))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key = 'value1')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: EXPLAIN SELECT * FROM src where key="value1"
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT * FROM src where key="value1"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) "value1"))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key = 'value1')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: EXPLAIN SELECT * FROM src where key="1"
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT * FROM src where key="1"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) "1"))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key = '1')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+hivevar:a=1
+hivevar:b=a
+hivevar:c=1
+hivevar:key1=value1