You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/11/20 20:33:35 UTC
svn commit: r1640791 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
itests/hive-unit/src/test/java/org/apache/hive/jdbc/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ ql/src/java/org/apac...
Author: szehon
Date: Thu Nov 20 19:33:34 2014
New Revision: 1640791
URL: http://svn.apache.org/r1640791
Log:
HIVE-8893 : Implement whitelist for builtin UDFs to avoid untrused code execution in multiuser mode (Prasad Mujumdar via Szehon)
Added:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestBlockedUdf.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java
hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Nov 20 19:33:34 2014
@@ -1752,6 +1752,13 @@ public class HiveConf extends Configurat
"If set to true (default), the logged-in user determines the fair scheduler queue\n" +
"for submitted jobs, so that map reduce resource usage can be tracked by user.\n" +
"If set to false, all Hive jobs go to the 'hive' user's queue."),
+ HIVE_SERVER2_BUILTIN_UDF_WHITELIST("hive.server2.builtin.udf.whitelist", "",
+ "Comma separated list of builtin udf names allowed in queries.\n" +
+ "An empty whitelist allows all builtin udfs to be executed. " +
+ " The udf black list takes precedence over udf white list"),
+ HIVE_SERVER2_BUILTIN_UDF_BLACKLIST("hive.server2.builtin.udf.blacklist", "",
+ "Comma separated list of udfs names. These udfs will not be allowed in queries." +
+ " The udf black list takes precedence over udf white list"),
HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile",
"Comma separated list of non-SQL Hive commands users are authorized to execute"),
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java Thu Nov 20 19:33:34 2014
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -30,12 +31,14 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.After;
import org.junit.AfterClass;
@@ -387,6 +390,110 @@ public class TestJdbcWithMiniHS2 {
verifyScratchDir(conf, fs, scratchDirPath, expectedFSPermission, userName, true);
}
+ /** Test UDF whitelist
+ * - verify default value
+ * - verify udf allowed with default whitelist
+ * - verify udf allowed with specific whitelist
+ * - verify udf disallowed when not in whitelist
+ * @throws Exception
+ */
+ @Test
+ public void testUdfWhiteList() throws Exception {
+ HiveConf testConf = new HiveConf();
+ assertTrue(testConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST).isEmpty());
+ // verify that udf in default whitelist can be executed
+ Statement stmt = hs2Conn.createStatement();
+ stmt.executeQuery("SELECT substr('foobar', 4) ");
+ hs2Conn.close();
+ miniHS2.stop();
+
+ // setup whitelist
+ Set<String> funcNames = FunctionRegistry.getFunctionNames();
+ funcNames.remove("reflect");
+ String funcNameStr = "";
+ for (String funcName : funcNames) {
+ funcNameStr += "," + funcName;
+ }
+ funcNameStr = funcNameStr.substring(1); // remove ',' at begining
+ testConf.setVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST, funcNameStr);
+ miniHS2 = new MiniHS2(testConf);
+ miniHS2.start(new HashMap<String, String>());
+
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+ stmt = hs2Conn.createStatement();
+ // verify that udf in whitelist can be executed
+ stmt.executeQuery("SELECT substr('foobar', 3) ");
+
+ // verify that udf not in whitelist fails
+ try {
+ stmt.executeQuery("SELECT reflect('java.lang.String', 'valueOf', 1) ");
+ fail("reflect() udf invocation should fail");
+ } catch (SQLException e) {
+ // expected
+ }
+ }
+
+ /** Test UDF blacklist
+ * - verify default value
+ * - verify udfs allowed with default blacklist
+ * - verify udf disallowed when in blacklist
+ * @throws Exception
+ */
+ @Test
+ public void testUdfBlackList() throws Exception {
+ HiveConf testConf = new HiveConf();
+ assertTrue(testConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST).isEmpty());
+
+ Statement stmt = hs2Conn.createStatement();
+ // verify that udf in default whitelist can be executed
+ stmt.executeQuery("SELECT substr('foobar', 4) ");
+
+ miniHS2.stop();
+ testConf.setVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST, "reflect");
+ miniHS2 = new MiniHS2(testConf);
+ miniHS2.start(new HashMap<String, String>());
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+ stmt = hs2Conn.createStatement();
+
+ try {
+ stmt.executeQuery("SELECT reflect('java.lang.String', 'valueOf', 1) ");
+ fail("reflect() udf invocation should fail");
+ } catch (SQLException e) {
+ // expected
+ }
+ }
+
+ /** Test UDF blacklist overrides whitelist
+ * @throws Exception
+ */
+ @Test
+ public void testUdfBlackListOverride() throws Exception {
+ // setup whitelist
+ HiveConf testConf = new HiveConf();
+
+ Set<String> funcNames = FunctionRegistry.getFunctionNames();
+ String funcNameStr = "";
+ for (String funcName : funcNames) {
+ funcNameStr += "," + funcName;
+ }
+ funcNameStr = funcNameStr.substring(1); // remove ',' at begining
+ testConf.setVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST, funcNameStr);
+ testConf.setVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST, "reflect");
+ miniHS2 = new MiniHS2(testConf);
+ miniHS2.start(new HashMap<String, String>());
+
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+ Statement stmt = hs2Conn.createStatement();
+
+ // verify that udf in black list fails even though it's included in whitelist
+ try {
+ stmt.executeQuery("SELECT reflect('java.lang.String', 'valueOf', 1) ");
+ fail("reflect() udf invocation should fail");
+ } catch (SQLException e) {
+ // expected
+ }
+ }
+
/**
* Tests the creation of the root hdfs scratch dir, which should be writable by all.
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java Thu Nov 20 19:33:34 2014
@@ -47,6 +47,8 @@ public class FunctionInfo implements Com
private Class<? extends TableFunctionResolver> tableFunctionResolver;
+ private boolean blockedFunction;
+
public FunctionInfo(boolean isNative, String displayName,
GenericUDF genericUDF) {
this.isNative = isNative;
@@ -190,4 +192,13 @@ public class FunctionInfo implements Com
public boolean isTableFunction() {
return null != tableFunctionResolver;
}
+
+ public boolean isBlockedFunction() {
+ return blockedFunction;
+ }
+
+ public void setBlockedFunction(boolean blockedFunction) {
+ this.blockedFunction = blockedFunction;
+ }
+
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Thu Nov 20 19:33:34 2014
@@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
@@ -41,6 +42,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@@ -148,6 +150,9 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
/**
* FunctionRegistry.
@@ -668,8 +673,12 @@ public final class FunctionRegistry {
return functionInfo;
}
- public static FunctionInfo getFunctionInfo(String functionName) {
- return getFunctionInfo(mFunctions, functionName);
+ public static FunctionInfo getFunctionInfo(String functionName) throws SemanticException {
+ FunctionInfo functionInfo = getFunctionInfo(mFunctions, functionName);
+ if (functionInfo != null && functionInfo.isBlockedFunction()) {
+ throw new SemanticException ("UDF " + functionName + " is not allowed");
+ }
+ return functionInfo;
}
/**
@@ -771,7 +780,13 @@ public final class FunctionRegistry {
public static Set<String> getFunctionSynonyms(String funcName) {
Set<String> synonyms = new HashSet<String>();
- FunctionInfo funcInfo = getFunctionInfo(funcName);
+ FunctionInfo funcInfo;
+ try {
+ funcInfo = getFunctionInfo(funcName);
+ } catch (SemanticException e) {
+ LOG.warn("Failed to load " + funcName);
+ funcInfo = null;
+ }
if (null == funcInfo) {
return synonyms;
}
@@ -1246,7 +1261,7 @@ public final class FunctionRegistry {
}
}
- public static GenericUDAFResolver getGenericUDAFResolver(String functionName) {
+ public static GenericUDAFResolver getGenericUDAFResolver(String functionName) throws SemanticException {
if (LOG.isDebugEnabled()) {
LOG.debug("Looking up GenericUDAF: " + functionName);
}
@@ -1543,16 +1558,18 @@ public final class FunctionRegistry {
/**
* A shortcut to get the "index" GenericUDF. This is used for getting elements
* out of array and getting values out of map.
+ * @throws SemanticException
*/
public static GenericUDF getGenericUDFForIndex() {
- return FunctionRegistry.getFunctionInfo("index").getGenericUDF();
+ return FunctionRegistry.getFunctionInfo(mFunctions, "index").getGenericUDF();
}
/**
* A shortcut to get the "and" GenericUDF.
+ * @throws SemanticException
*/
public static GenericUDF getGenericUDFForAnd() {
- return FunctionRegistry.getFunctionInfo("and").getGenericUDF();
+ return FunctionRegistry.getFunctionInfo(mFunctions, "and").getGenericUDF();
}
/**
@@ -1924,8 +1941,9 @@ public final class FunctionRegistry {
* name of function
* @return true if a GenericUDF or GenericUDAF exists for this name and implyOrder is true, false
* otherwise.
+ * @throws SemanticException
*/
- public static boolean impliesOrder(String functionName) {
+ public static boolean impliesOrder(String functionName) throws SemanticException {
FunctionInfo info = getFunctionInfo(functionName);
if (info != null) {
@@ -1951,13 +1969,13 @@ public final class FunctionRegistry {
windowFunctions.put(functionName.toLowerCase(), wInfo);
}
- public static boolean isTableFunction(String name)
+ public static boolean isTableFunction(String name) throws SemanticException
{
FunctionInfo tFInfo = getFunctionInfo(name);
return tFInfo != null && !tFInfo.isInternalTableFunction() && tFInfo.isTableFunction();
}
- public static TableFunctionResolver getTableFunctionResolver(String name)
+ public static TableFunctionResolver getTableFunctionResolver(String name) throws SemanticException
{
FunctionInfo tfInfo = getFunctionInfo(name);
if(tfInfo.isTableFunction()) {
@@ -1966,7 +1984,7 @@ public final class FunctionRegistry {
return null;
}
- public static TableFunctionResolver getWindowingTableFunction()
+ public static TableFunctionResolver getWindowingTableFunction() throws SemanticException
{
return getTableFunctionResolver(WINDOWING_TABLE_FUNCTION);
}
@@ -1993,8 +2011,9 @@ public final class FunctionRegistry {
* name of a function
* @return true if function is a UDAF, has WindowFunctionDescription annotation and the annotations
* confirms a ranking function, false otherwise
+ * @throws SemanticException
*/
- public static boolean isRankingFunction(String name) {
+ public static boolean isRankingFunction(String name) throws SemanticException {
FunctionInfo info = getFunctionInfo(name);
if (info == null) {
return false;
@@ -2026,4 +2045,57 @@ public final class FunctionRegistry {
}
nativeUdfs.add(fi.getFunctionClass());
}
+
+ /**
+ * Setup blocked flag for all builtin UDFs as per udf whitelist and blacklist
+ * @param whiteList
+ * @param blackList
+ */
+ public static void setupPermissionsForBuiltinUDFs(String whiteListStr,
+ String blackListStr) {
+ List<String> whiteList = Lists.newArrayList(
+ Splitter.on(",").trimResults().omitEmptyStrings().split(whiteListStr));
+ List<String> blackList = Lists.newArrayList(
+ Splitter.on(",").trimResults().omitEmptyStrings().split(blackListStr));
+
+ for ( Entry<String, FunctionInfo> funcEntry : mFunctions.entrySet()) {
+ funcEntry.getValue().setBlockedFunction(
+ isUdfBlocked(funcEntry.getKey(), whiteList, blackList));
+ }
+ }
+
+ /**
+ * Check if the function belongs to whitelist or blacklist
+ * @param functionName
+ * @param whiteList
+ * @param blackList
+ * @return true if the given udf is to be blocked
+ */
+ private static boolean isUdfBlocked(String functionName,
+ List<String> whiteList, List<String> blackList) {
+ boolean inWhiteList = false;
+ boolean inBlackList = false;
+
+ if (whiteList.isEmpty()) {
+ // if whitelist is empty, all udfs are allowed
+ inWhiteList = true;
+ } else {
+ for (String allowedFunction : whiteList) {
+ if (functionName.equalsIgnoreCase(allowedFunction)) {
+ inWhiteList = true;
+ break;
+ }
+ }
+ }
+
+ for (String blockedFunction : blackList) {
+ if (functionName.equalsIgnoreCase(blockedFunction)) {
+ inBlackList = true;
+ break;
+ }
+ }
+
+ // blacklist setting takes presendence on whitelist
+ return !inWhiteList || inBlackList;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Thu Nov 20 19:33:34 2014
@@ -82,6 +82,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor;
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFArgDesc;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -756,7 +757,13 @@ public class VectorizationContext {
if (udfName == null) {
return false;
}
- FunctionInfo funcInfo = FunctionRegistry.getFunctionInfo(udfName);
+ FunctionInfo funcInfo;
+ try {
+ funcInfo = FunctionRegistry.getFunctionInfo(udfName);
+ } catch (SemanticException e) {
+ LOG.warn("Failed to load " + udfName, e);
+ funcInfo = null;
+ }
if (funcInfo == null) {
return false;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Thu Nov 20 19:33:34 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.SettableUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
@@ -111,13 +112,24 @@ public class SqlFunctionConverter {
name = FunctionRegistry.UNARY_MINUS_FUNC_NAME;
}
}
- FunctionInfo hFn = name != null ? FunctionRegistry.getFunctionInfo(name) : null;
+ FunctionInfo hFn;
+ try {
+ hFn = name != null ? FunctionRegistry.getFunctionInfo(name) : null;
+ } catch (SemanticException e) {
+ LOG.warn("Failed to load udf " + name, e);
+ hFn = null;
+ }
if (hFn == null)
- hFn = handleExplicitCast(op, dt);
+ try {
+ hFn = handleExplicitCast(op, dt);
+ } catch (SemanticException e) {
+ LOG.warn("Failed to load udf " + name, e);
+ hFn = null;
+ }
return hFn == null ? null : hFn.getGenericUDF();
}
- private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) {
+ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) throws SemanticException {
FunctionInfo castUDF = null;
if (op.kind == SqlKind.CAST) {
@@ -283,7 +295,13 @@ public class SqlFunctionConverter {
private void registerFunction(String name, SqlOperator optiqFn, HiveToken hiveToken) {
reverseOperatorMap.put(optiqFn, name);
- FunctionInfo hFn = FunctionRegistry.getFunctionInfo(name);
+ FunctionInfo hFn;
+ try {
+ hFn = FunctionRegistry.getFunctionInfo(name);
+ } catch (SemanticException e) {
+ LOG.warn("Failed to load udf " + name, e);
+ hFn = null;
+ }
if (hFn != null) {
String hFnName = getName(hFn.getGenericUDF());
hiveToOptiq.put(hFnName, optiqFn);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Thu Nov 20 19:33:34 2014
@@ -3020,9 +3020,9 @@ public class DDLSemanticAnalyzer extends
}
private static ExprNodeGenericFuncDesc makeBinaryPredicate(
- String fn, ExprNodeDesc left, ExprNodeDesc right) {
- return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
- FunctionRegistry.getFunctionInfo(fn).getGenericUDF(), Lists.newArrayList(left, right));
+ String fn, ExprNodeDesc left, ExprNodeDesc right) throws SemanticException {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+ FunctionRegistry.getFunctionInfo(fn).getGenericUDF(), Lists.newArrayList(left, right));
}
/**
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java Thu Nov 20 19:33:34 2014
@@ -59,7 +59,7 @@ public class FunctionSemanticAnalyzer ex
analyzeCreateFunction(ast);
}
if (ast.getToken().getType() == HiveParser.TOK_DROPFUNCTION) {
- analyzeDropFunction(ast);
+ analyzeDropFunction(ast);
}
LOG.info("analyze done");
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Thu Nov 20 19:33:34 2014
@@ -690,7 +690,12 @@ public class TypeCheckProcFactory {
static ExprNodeDesc getFuncExprNodeDescWithUdfData(String udfName, TypeInfo typeInfo,
ExprNodeDesc... children) throws UDFArgumentException {
- FunctionInfo fi = FunctionRegistry.getFunctionInfo(udfName);
+ FunctionInfo fi;
+ try {
+ fi = FunctionRegistry.getFunctionInfo(udfName);
+ } catch (SemanticException e) {
+ throw new UDFArgumentException(e);
+ }
if (fi == null) {
throw new UDFArgumentException(udfName + " not found.");
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java Thu Nov 20 19:33:34 2014
@@ -189,11 +189,11 @@ public class TestMetastoreExpr extends T
return (ExprNodeGenericFuncDesc)stack.pop();
}
- public ExprBuilder pred(String name, int args) {
+ public ExprBuilder pred(String name, int args) throws Exception {
return fn(name, TypeInfoFactory.booleanTypeInfo, args);
}
- private ExprBuilder fn(String name, TypeInfo ti, int args) {
+ private ExprBuilder fn(String name, TypeInfo ti, int args) throws Exception {
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
for (int i = 0; i < args; ++i) {
children.add(stack.pop());
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Thu Nov 20 19:33:34 2014
@@ -181,7 +181,7 @@ public class TestExecDriver extends Test
}
}
- private FilterDesc getTestFilterDesc(String column) {
+ private FilterDesc getTestFilterDesc(String column) throws Exception {
ArrayList<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>();
children1.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
column, "", false));
@@ -208,7 +208,7 @@ public class TestExecDriver extends Test
}
@SuppressWarnings("unchecked")
- private void populateMapPlan1(Table src) {
+ private void populateMapPlan1(Table src) throws Exception {
Operator<FileSinkDesc> op2 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator
+ "mapplan1.out"), Utilities.defaultTd, true));
@@ -219,7 +219,7 @@ public class TestExecDriver extends Test
}
@SuppressWarnings("unchecked")
- private void populateMapPlan2(Table src) {
+ private void populateMapPlan2(Table src) throws Exception {
Operator<FileSinkDesc> op3 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator
+ "mapplan2.out"), Utilities.defaultTd, false));
@@ -267,7 +267,7 @@ public class TestExecDriver extends Test
}
@SuppressWarnings("unchecked")
- private void populateMapRedPlan2(Table src) throws SemanticException {
+ private void populateMapRedPlan2(Table src) throws Exception {
ArrayList<String> outputColumns = new ArrayList<String>();
for (int i = 0; i < 2; i++) {
outputColumns.add("_col" + i);
@@ -423,7 +423,7 @@ public class TestExecDriver extends Test
}
@SuppressWarnings("unchecked")
- private void populateMapRedPlan6(Table src) throws SemanticException {
+ private void populateMapRedPlan6(Table src) throws Exception {
// map-side work
ArrayList<String> outputColumns = new ArrayList<String>();
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExpressionEvaluator.java Thu Nov 20 19:33:34 2014
@@ -122,12 +122,12 @@ public class TestExpressionEvaluator ext
}
}
- private static ExprNodeDesc getListIndexNode(ExprNodeDesc node, int index) {
+ private static ExprNodeDesc getListIndexNode(ExprNodeDesc node, int index) throws Exception {
return getListIndexNode(node, new ExprNodeConstantDesc(index));
}
private static ExprNodeDesc getListIndexNode(ExprNodeDesc node,
- ExprNodeDesc index) {
+ ExprNodeDesc index) throws Exception {
ArrayList<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(2);
children.add(node);
children.add(index);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java Thu Nov 20 19:33:34 2014
@@ -28,6 +28,7 @@ import junit.framework.TestCase;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
@@ -379,7 +380,7 @@ public class TestFunctionRegistry extend
protected void tearDown() {
}
- public void testIsRankingFunction() {
+ public void testIsRankingFunction() throws Exception {
Assert.assertTrue(FunctionRegistry.isRankingFunction("rank"));
Assert.assertTrue(FunctionRegistry.isRankingFunction("dense_rank"));
Assert.assertTrue(FunctionRegistry.isRankingFunction("percent_rank"));
@@ -387,7 +388,7 @@ public class TestFunctionRegistry extend
Assert.assertFalse(FunctionRegistry.isRankingFunction("min"));
}
- public void testImpliesOrder() {
+ public void testImpliesOrder() throws Exception {
Assert.assertTrue(FunctionRegistry.impliesOrder("rank"));
Assert.assertTrue(FunctionRegistry.impliesOrder("dense_rank"));
Assert.assertTrue(FunctionRegistry.impliesOrder("percent_rank"));
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestBlockedUdf.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestBlockedUdf.java?rev=1640791&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestBlockedUdf.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestBlockedUdf.java Thu Nov 20 19:33:34 2014
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf;
+
+import static org.junit.Assert.*;
+
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestBlockedUdf {
+
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ FunctionRegistry.setupPermissionsForBuiltinUDFs("", "");
+ }
+
+ /**
+ * Verify that UDF in the whitelist can be access
+ * @throws Exception
+ */
+ @Test
+ public void testDefaultWhiteList() throws Exception {
+ assertEquals("", new HiveConf().getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST));
+ assertEquals("", new HiveConf().getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST));
+ FunctionRegistry.setupPermissionsForBuiltinUDFs("", "");
+ assertEquals("substr", FunctionRegistry.getFunctionInfo("substr").getDisplayName());
+ }
+
+ /**
+ * Verify that UDF in the whitelist can be access
+ * @throws Exception
+ */
+ @Test
+ public void testUdfInWhiteList() throws Exception {
+ Set<String> funcNames = FunctionRegistry.getFunctionNames();
+ funcNames.remove("reflect");
+ FunctionRegistry.setupPermissionsForBuiltinUDFs(funcNames.toString(), "");
+ assertEquals("substr", FunctionRegistry.getFunctionInfo("substr").getDisplayName());
+ }
+
+ /**
+ * Verify that UDF not in whitelist can't be accessed
+ * @throws Exception
+ */
+ @Test (expected=SemanticException.class)
+ public void testUdfNotInWhiteList() throws Exception {
+ Set<String> funcNames = FunctionRegistry.getFunctionNames();
+ funcNames.remove("reflect");
+ FunctionRegistry.setupPermissionsForBuiltinUDFs(funcNames.toString(), "");
+ assertEquals("reflect", FunctionRegistry.getFunctionInfo("reflect").getDisplayName());
+ }
+
+ /**
+ * Verify that UDF in blacklist can't be accessed
+ * @throws Exception
+ */
+ @Test (expected=SemanticException.class)
+ public void testUdfInBlackList() throws Exception {
+ FunctionRegistry.setupPermissionsForBuiltinUDFs("", "reflect");
+ assertEquals("reflect", FunctionRegistry.getFunctionInfo("reflect").getDisplayName());
+ }
+
+ /**
+ * Verify that UDF in whitelist and blacklist can't be accessed
+ * @throws Exception
+ */
+ @Test (expected=SemanticException.class)
+ public void testUdfInBlackAndWhiteList() throws Exception {
+ FunctionRegistry.setupPermissionsForBuiltinUDFs("reflect", "reflect");
+ assertEquals("reflect", FunctionRegistry.getFunctionInfo("reflect").getDisplayName());
+ }
+
+ /**
+ * Test malformatted udf list setting
+ */
+ @Test (expected=SemanticException.class)
+ public void testMalformattedListProperty() throws Exception {
+ FunctionRegistry.setupPermissionsForBuiltinUDFs(",,", " ,reflect,");
+ assertEquals("reflect", FunctionRegistry.getFunctionInfo("reflect").getDisplayName());
+ }
+
+}
Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1640791&r1=1640790&r2=1640791&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java Thu Nov 20 19:33:34 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -46,6 +47,9 @@ import org.apache.hive.service.cli.sessi
import org.apache.hive.service.cli.thrift.TProtocolVersion;
import org.apache.hive.service.server.HiveServer2;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
/**
* CLIService.
*
@@ -104,9 +108,16 @@ public class CLIService extends Composit
}
}
}
+ setupBlockedUdfs();
super.init(hiveConf);
}
+ private void setupBlockedUdfs() {
+ FunctionRegistry.setupPermissionsForBuiltinUDFs(
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_WHITELIST),
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_BUILTIN_UDF_BLACKLIST));
+ }
+
public UserGroupInformation getServiceUGI() {
return this.serviceUGI;
}