You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2014/09/29 19:36:13 UTC
svn commit: r1628229 - in /hive/trunk:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/test/results/clientnegative/ ql/src/test/results/clientpositive/
Author: prasadm
Date: Mon Sep 29 17:36:13 2014
New Revision: 1628229
URL: http://svn.apache.org/r1628229
Log:
HIVE-8266: create function using <resource> statement compilation should include resource URI entity (Prasad Mujumdar reviewed by Brock Noland)
Added:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
hive/trunk/ql/src/test/results/clientnegative/udf_local_resource.q.out
hive/trunk/ql/src/test/results/clientnegative/udf_nonexistent_resource.q.out
hive/trunk/ql/src/test/results/clientpositive/udf_using.q.out
Added: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java?rev=1628229&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java (added)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java Mon Sep 29 17:36:13 2014
@@ -0,0 +1,74 @@
+package org.apache.hadoop.hive.ql;
+
+import static org.junit.Assert.*;
+
+import java.net.URI;
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.hooks.Entity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCreateUdfEntities {
+ private Driver driver;
+ private String funcName = "print_test";
+
+ @Before
+ public void setUp() throws Exception {
+
+ HiveConf conf = new HiveConf(Driver.class);
+ SessionState.start(conf);
+ driver = new Driver(conf);
+ driver.init();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ driver.run("drop function " + funcName);
+ driver.close();
+ SessionState.get().close();
+ }
+
+ @Test
+ public void testUdfWithLocalResource() throws Exception {
+ int rc = driver.compile("CREATE FUNCTION " + funcName + " AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFPrintf' "
+ + " using file '" + "file:///tmp/udf1.jar'");
+ assertEquals(0, rc);
+ WriteEntity outputEntities[] = driver.getPlan().getOutputs().toArray(new WriteEntity[] {});
+ assertEquals(outputEntities.length, 3);
+
+ assertEquals(Entity.Type.DATABASE, outputEntities[0].getType());
+ assertEquals("default", outputEntities[0].getDatabase().getName());
+
+ assertEquals(Entity.Type.FUNCTION, outputEntities[1].getType());
+ assertEquals(funcName, outputEntities[1].getFunctionName());
+
+ assertEquals(Entity.Type.LOCAL_DIR, outputEntities[2].getType());
+ assertEquals("file:/tmp/udf1.jar", outputEntities[2].getLocation().toString());
+ }
+
+ @Test
+ public void testUdfWithDfsResource() throws Exception {
+ int rc = driver.compile("CREATE FUNCTION default." + funcName + " AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFPrintf' "
+ + " using file '" + "hdfs:///tmp/udf1.jar'");
+ assertEquals(0, rc);
+ WriteEntity outputEntities[] = driver.getPlan().getOutputs().toArray(new WriteEntity[] {});
+ assertEquals(outputEntities.length, 3);
+
+ assertEquals(Entity.Type.DATABASE, outputEntities[0].getType());
+ assertEquals("default", outputEntities[0].getDatabase().getName());
+
+ assertEquals(Entity.Type.FUNCTION, outputEntities[1].getType());
+ assertEquals(funcName, outputEntities[1].getFunctionName());
+
+ assertEquals(Entity.Type.DFS_DIR, outputEntities[2].getType());
+ assertEquals("hdfs:/tmp/udf1.jar", outputEntities[2].getLocation().toString());
+ }
+
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java?rev=1628229&r1=1628228&r2=1628229&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java Mon Sep 29 17:36:13 2014
@@ -22,6 +22,8 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -81,7 +83,7 @@ public class FunctionSemanticAnalyzer ex
new CreateFunctionDesc(functionName, isTemporaryFunction, className, resources);
rootTasks.add(TaskFactory.get(new FunctionWork(desc), conf));
- addEntities(functionName, isTemporaryFunction);
+ addEntities(functionName, isTemporaryFunction, resources);
}
private void analyzeDropFunction(ASTNode ast) throws SemanticException {
@@ -106,7 +108,7 @@ public class FunctionSemanticAnalyzer ex
DropFunctionDesc desc = new DropFunctionDesc(functionName, isTemporaryFunction);
rootTasks.add(TaskFactory.get(new FunctionWork(desc), conf));
- addEntities(functionName, isTemporaryFunction);
+ addEntities(functionName, isTemporaryFunction, null);
}
private ResourceType getResourceType(ASTNode token) throws SemanticException {
@@ -152,8 +154,8 @@ public class FunctionSemanticAnalyzer ex
/**
* Add write entities to the semantic analyzer to restrict function creation to privileged users.
*/
- private void addEntities(String functionName, boolean isTemporaryFunction)
- throws SemanticException {
+ private void addEntities(String functionName, boolean isTemporaryFunction,
+ List<ResourceUri> resources) throws SemanticException {
// If the function is being added under a database 'namespace', then add an entity representing
// the database (only applicable to permanent/metastore functions).
// We also add a second entity representing the function name.
@@ -183,5 +185,13 @@ public class FunctionSemanticAnalyzer ex
// Add the function name as a WriteEntity
outputs.add(new WriteEntity(database, functionName, Type.FUNCTION,
WriteEntity.WriteType.DDL_NO_LOCK));
+
+ if (resources != null) {
+ for (ResourceUri resource : resources) {
+ String uriPath = resource.getUri();
+ outputs.add(new WriteEntity(new Path(uriPath),
+ FileUtils.isLocalFile(conf, uriPath)));
+ }
+ }
}
}
Modified: hive/trunk/ql/src/test/results/clientnegative/udf_local_resource.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_local_resource.q.out?rev=1628229&r1=1628228&r2=1628229&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_local_resource.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_local_resource.q.out Mon Sep 29 17:36:13 2014
@@ -1,5 +1,6 @@
PREHOOK: query: create function lookup as 'org.apache.hadoop.hive.ql.udf.UDFFileLookup' using file '../../data/files/sales.txt'
PREHOOK: type: CREATEFUNCTION
+#### A masked pattern was here ####
PREHOOK: Output: database:default
PREHOOK: Output: default.lookup
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.FunctionTask. Hive warehouse is non-local, but ../../data/files/sales.txt specifies file on local filesystem. Resources on non-local warehouse should specify a non-local scheme/path
Modified: hive/trunk/ql/src/test/results/clientnegative/udf_nonexistent_resource.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_nonexistent_resource.q.out?rev=1628229&r1=1628228&r2=1628229&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_nonexistent_resource.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_nonexistent_resource.q.out Mon Sep 29 17:36:13 2014
@@ -2,5 +2,6 @@ PREHOOK: query: create function lookup a
PREHOOK: type: CREATEFUNCTION
PREHOOK: Output: database:default
PREHOOK: Output: default.lookup
+PREHOOK: Output: nonexistent_file.txt
nonexistent_file.txt does not exist
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.FunctionTask. nonexistent_file.txt does not exist
Modified: hive/trunk/ql/src/test/results/clientpositive/udf_using.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_using.q.out?rev=1628229&r1=1628228&r2=1628229&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_using.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_using.q.out Mon Sep 29 17:36:13 2014
@@ -6,6 +6,7 @@ PREHOOK: Output: default.lookup
POSTHOOK: type: CREATEFUNCTION
POSTHOOK: Output: database:default
POSTHOOK: Output: default.lookup
+#### A masked pattern was here ####
PREHOOK: query: create table udf_using (c1 string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default