You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2016/03/09 09:47:39 UTC
hive git commit: HIVE-11483 : Add encoding and decoding for query
string config (Rajat Khandelwal, reviewed by Amareshwari Sriramadasu
Repository: hive
Updated Branches:
refs/heads/master fe14a9088 -> 415373bbd
HIVE-11483 : Add encoding and decoding for query string config (Rajat Khandelwal, reviewed by Amareshwari Sriramadasu
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/415373bb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/415373bb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/415373bb
Branch: refs/heads/master
Commit: 415373bbd1c58e7b8f00aa32c0bb7d485d4f91e5
Parents: fe14a90
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Mar 9 14:17:21 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Mar 9 14:17:21 2016 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 76 ++++++++++++++++----
.../apache/hadoop/hive/conf/TestHiveConf.java | 10 +++
.../java/org/apache/hadoop/hive/ql/Driver.java | 5 +-
.../hive/ql/exec/errors/TaskLogProcessor.java | 2 +-
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 2 +-
.../hadoop/hive/ql/exec/tez/DagUtils.java | 2 +-
.../hadoop/hive/ql/session/SessionState.java | 6 +-
.../ql/exec/errors/TestTaskLogProcessor.java | 8 +--
.../apache/hadoop/hive/ql/hooks/TestHooks.java | 2 +-
9 files changed, 85 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5098851..27a56dd 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -40,12 +40,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.security.auth.login.LoginException;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
+
+import java.io.*;
import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -96,6 +95,35 @@ public class HiveConf extends Configuration {
this.isSparkConfigUpdated = isSparkConfigUpdated;
}
+ public interface EncoderDecoder<K, V> {
+ V encode(K key);
+ K decode(V value);
+ }
+
+ public static class URLEncoderDecoder implements EncoderDecoder<String, String> {
+ private static final String UTF_8 = "UTF-8";
+ @Override
+ public String encode(String key) {
+ try {
+ return URLEncoder.encode(key, UTF_8);
+ } catch (UnsupportedEncodingException e) {
+ return key;
+ }
+ }
+
+ @Override
+ public String decode(String value) {
+ try {
+ return URLDecoder.decode(value, UTF_8);
+ } catch (UnsupportedEncodingException e) {
+ return value;
+ }
+ }
+ }
+ public static class EncoderDecoderFactory {
+ public static final URLEncoderDecoder URL_ENCODER_DECODER = new URLEncoderDecoder();
+ }
+
static {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
if (classLoader == null) {
@@ -3283,10 +3311,8 @@ public class HiveConf extends Configuration {
public static String getVar(Configuration conf, ConfVars var) {
assert (var.valClass == String.class) : var.varname;
- if (var.altName != null) {
- return conf.get(var.varname, conf.get(var.altName, var.defaultStrVal));
- }
- return conf.get(var.varname, var.defaultStrVal);
+ return var.altName != null ? conf.get(var.varname, conf.get(var.altName, var.defaultStrVal))
+ : conf.get(var.varname, var.defaultStrVal);
}
public static String getTrimmedVar(Configuration conf, ConfVars var) {
@@ -3309,10 +3335,13 @@ public class HiveConf extends Configuration {
}
public static String getVar(Configuration conf, ConfVars var, String defaultVal) {
- if (var.altName != null) {
- return conf.get(var.varname, conf.get(var.altName, defaultVal));
- }
- return conf.get(var.varname, defaultVal);
+ String ret = var.altName != null ? conf.get(var.varname, conf.get(var.altName, defaultVal))
+ : conf.get(var.varname, defaultVal);
+ return ret;
+ }
+
+ public static String getVar(Configuration conf, ConfVars var, EncoderDecoder<String, String> encoderDecoder) {
+ return encoderDecoder.decode(getVar(conf, var));
}
public String getLogIdVar(String defaultValue) {
@@ -3333,6 +3362,10 @@ public class HiveConf extends Configuration {
assert (var.valClass == String.class) : var.varname;
conf.set(var.varname, val);
}
+ public static void setVar(Configuration conf, ConfVars var, String val,
+ EncoderDecoder<String, String> encoderDecoder) {
+ setVar(conf, var, encoderDecoder.encode(val));
+ }
public static ConfVars getConfVars(String name) {
return vars.get(name);
@@ -3350,6 +3383,21 @@ public class HiveConf extends Configuration {
setVar(this, var, val);
}
+ public String getQueryString() {
+ return getQueryString(this);
+ }
+
+ public static String getQueryString(Configuration conf) {
+ return getVar(conf, ConfVars.HIVEQUERYSTRING, EncoderDecoderFactory.URL_ENCODER_DECODER);
+ }
+
+ public void setQueryString(String query) {
+ setQueryString(this, query);
+ }
+
+ public static void setQueryString(Configuration conf, String query) {
+ setVar(conf, ConfVars.HIVEQUERYSTRING, query, EncoderDecoderFactory.URL_ENCODER_DECODER);
+ }
public void logVars(PrintStream ps) {
for (ConfVars one : ConfVars.values()) {
ps.println(one.varname + "=" + ((get(one.varname) != null) ? get(one.varname) : ""));
@@ -3904,7 +3952,7 @@ public class HiveConf extends Configuration {
}
private static boolean isAllowed(Configuration conf, ConfVars setting) {
- String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, null);
+ String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, (String)null);
return (mode != null) ? !"strict".equals(mode) : !HiveConf.getBoolVar(conf, setting);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
index 365d500..f88573f 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
@@ -25,6 +25,8 @@ import org.apache.hive.common.util.HiveTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
import java.util.concurrent.TimeUnit;
@@ -156,4 +158,12 @@ public class TestHiveConf {
conf.setSparkConfigUpdated(false);
Assert.assertFalse(conf.getSparkConfigUpdated());
}
+ @Test
+ public void testEncodingDecoding() throws UnsupportedEncodingException {
+ HiveConf conf = new HiveConf();
+ String query = "select blah, '\u0001' from random_table";
+ conf.setQueryString(query);
+ Assert.assertEquals(URLEncoder.encode(query, "UTF-8"), conf.get(ConfVars.HIVEQUERYSTRING.varname));
+ Assert.assertEquals(query, conf.getQueryString());
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3253146..b50c5a2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -86,7 +86,6 @@ import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.CalcitePlanner;
import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
@@ -499,7 +498,7 @@ public class Driver implements CommandProcessor {
plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
SessionState.get().getHiveOperation(), schema);
- conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);
+ conf.setQueryString(queryStr);
conf.set("mapreduce.workflow.id", "hive_" + queryId);
conf.set("mapreduce.workflow.name", queryStr);
@@ -1484,7 +1483,7 @@ public class Driver implements CommandProcessor {
String queryId = plan.getQueryId();
// Get the query string from the conf file as the compileInternal() method might
// hide sensitive information during query redaction.
- String queryStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
+ String queryStr = conf.getQueryString();
maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER);
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
index 68123d4..b788d8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
@@ -53,7 +53,7 @@ public class TaskLogProcessor {
private final String query;
public TaskLogProcessor(JobConf conf) {
- query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
+ query = HiveConf.getQueryString(conf);
heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats());
heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats());
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index b184b4e..ce020a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -568,7 +568,7 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
// Intentionally overwrites anything the user may have put here
conf.setBoolean("hive.input.format.sorted", mWork.isInputFormatSorted());
- if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, null) == null) {
+ if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, (String)null) == null) {
HiveConf.setVar(conf, ConfVars.HIVE_CURRENT_DATABASE, getCurrentDB());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 473dbd6..79da860 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -904,7 +904,7 @@ public class DagUtils {
public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException {
FileStatus fstatus = null;
- String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, null);
+ String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, (String)null);
if (hdfsDirPathStr != null) {
LOG.info("Hive jar directory is " + hdfsDirPathStr);
fstatus = validateTargetDir(new Path(hdfsDirPathStr), conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 109cd8c..78bbb1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -373,11 +373,11 @@ public class SessionState {
}
public void setCmd(String cmdString) {
- conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, cmdString);
+ conf.setQueryString(cmdString);
}
public String getCmd() {
- return (conf.getVar(HiveConf.ConfVars.HIVEQUERYSTRING));
+ return (conf.getQueryString());
}
public String getQueryId() {
@@ -1642,7 +1642,7 @@ public class SessionState {
// Provide a facility to set current timestamp during tests
if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) {
String overrideTimestampString =
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, null);
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, (String)null);
if (overrideTimestampString != null && overrideTimestampString.length() > 0) {
queryCurrentTimestamp = Timestamp.valueOf(overrideTimestampString);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
index 67a86a6..477479d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
@@ -98,7 +98,7 @@ public class TestTaskLogProcessor {
@Test
public void testGetStackTraces() throws Exception {
JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");
final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);
@@ -150,7 +150,7 @@ public class TestTaskLogProcessor {
@Test
public void testScriptErrorHeuristic() throws Exception {
JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");
final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);
@@ -177,7 +177,7 @@ public class TestTaskLogProcessor {
@Test
public void testDataCorruptErrorHeuristic() throws Exception {
JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");
final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);
@@ -210,7 +210,7 @@ public class TestTaskLogProcessor {
@Test
public void testMapAggrMemErrorHeuristic() throws Exception {
JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");
final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);
http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
index 8d27762..4c14d8b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
@@ -73,7 +73,7 @@ public class TestHooks {
Driver driver = createDriver(conf);
int ret = driver.compile("select 'XXX' from t1");
assertEquals("Checking command success", 0, ret);
- assertEquals("select 'AAA' from t1", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING));
+ assertEquals("select 'AAA' from t1", conf.getQueryString());
}
public static class SimpleQueryRedactor extends Redactor {