You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2015/01/22 01:00:34 UTC
svn commit: r1653713 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/hooks/
ql/src/test/org/apache/hadoop/hive/ql/hooks/
Author: brock
Date: Thu Jan 22 00:00:33 2015
New Revision: 1653713
URL: http://svn.apache.org/r1653713
Log:
HIVE-9408 - Add hook interface so queries can be redacted before being placed in job.xml (Brock reviewed by Xuefu)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1653713&r1=1653712&r2=1653713&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jan 22 00:00:33 2015
@@ -271,6 +271,10 @@ public class HiveConf extends Configurat
"Comma-separated list of on-failure hooks to be invoked for each statement. \n" +
"An on-failure hook is specified as the name of Java class which implements the \n" +
"org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext interface."),
+ QUERYREDACTORHOOKS("hive.exec.query.redactor.hooks", "",
+ "Comma-separated list of hooks to be invoked for each query which can \n" +
+ "tranform the query before it's placed in the job.xml file. Must be a Java class which \n" +
+ "extends from the org.apache.hadoop.hive.ql.hooks.Redactor abstract class."),
CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", "",
"Comma-separated list of statistics publishers to be invoked on counters on each job. \n" +
"A client stats publisher is specified as the name of a Java class which implements the \n" +
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1653713&r1=1653712&r2=1653713&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Thu Jan 22 00:00:33 2015
@@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.hooks.P
import org.apache.hadoop.hive.ql.hooks.PreExecute;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.hooks.Redactor;
import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
@@ -439,6 +440,11 @@ public class Driver implements CommandPr
SessionState.get().getCommandType());
String queryStr = plan.getQueryStr();
+ List<Redactor> queryRedactors = getHooks(ConfVars.QUERYREDACTORHOOKS, Redactor.class);
+ for (Redactor redactor : queryRedactors) {
+ redactor.setConf(conf);
+ queryStr = redactor.redactQuery(queryStr);
+ }
conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);
conf.set("mapreduce.workflow.id", "hive_" + queryId);
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java?rev=1653713&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/Redactor.java Thu Jan 22 00:00:33 2015
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class Redactor implements Hook, Configurable {
+
+ private Configuration conf;
+
+ /**
+ * Guranteed to be called before redactQuery is called
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Implementations may modify the query so that when placed in the job.xml
+ * and thus potenially exposed to admin users, the query does not expose
+ * sensitive information.
+ */
+ public String redactQuery(String query) {
+ return query;
+ }
+}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java?rev=1653713&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java Thu Jan 22 00:00:33 2015
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.hooks;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+public class TestHooks {
+
+ @BeforeClass
+ public static void onetimeSetup() throws Exception {
+ HiveConf conf = new HiveConf(TestHooks.class);
+ Driver driver = createDriver(conf);
+ int ret = driver.run("create table t1(i int)").getResponseCode();
+ assertEquals("Checking command success", 0, ret);
+ }
+
+ @AfterClass
+ public static void onetimeTeardown() throws Exception {
+ HiveConf conf = new HiveConf(TestHooks.class);
+ Driver driver = createDriver(conf);
+ driver.run("drop table t1");
+ }
+
+ @Before
+ public void setup() {
+ }
+
+ @Test
+ public void testQueryRedactor() throws Exception {
+ HiveConf conf = new HiveConf(TestHooks.class);
+ HiveConf.setVar(conf, HiveConf.ConfVars.QUERYREDACTORHOOKS,
+ SimpleQueryRedactor.class.getName());
+ Driver driver = createDriver(conf);
+ int ret = driver.compile("select 'XXX' from t1");
+ assertEquals("Checking command success", 0, ret);
+ assertEquals("select 'AAA' from t1", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING));
+ }
+
+ public static class SimpleQueryRedactor extends Redactor {
+ public String redactQuery(String query) {
+ return query.replaceAll("XXX", "AAA");
+ }
+ }
+
+ private static Driver createDriver(HiveConf conf) {
+ HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ SessionState.start(conf);
+ Driver driver = new Driver(conf);
+ driver.init();
+ return driver;
+ }
+
+}