You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/08/01 03:59:52 UTC
svn commit: r1615022 - in /hive/trunk:
itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/
itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/hooks/ service/s...
Author: szehon
Date: Fri Aug 1 01:59:52 2014
New Revision: 1615022
URL: http://svn.apache.org/r1615022
Log:
HIVE-7547 : Add ipAddress and userName to ExecHook (Szehon, reviewed by Thejas)
Added:
hive/trunk/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
Added: hive/trunk/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java?rev=1615022&view=auto
==============================================================================
--- hive/trunk/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java (added)
+++ hive/trunk/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java Fri Aug 1 01:59:52 2014
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.minikdc;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext.HookType;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests information retrieved from hooks, in Kerberos mode.
+ */
+public class TestHs2HooksWithMiniKdc {
+ public static class PostExecHook implements ExecuteWithHookContext {
+ public static String userName = null;
+ public static String ipAddress = null;
+
+ public void run(HookContext hookContext) {
+ if (hookContext.getHookType().equals(HookType.POST_EXEC_HOOK)) {
+ Assert.assertNotNull(hookContext.getIpAddress(), "IP Address is null");
+ ipAddress = hookContext.getIpAddress();
+ Assert.assertNotNull(hookContext.getUserName(), "Username is null");
+ userName = hookContext.getUserName();
+ }
+ }
+ }
+
+ public static class PreExecHook implements ExecuteWithHookContext {
+ public static String userName = null;
+ public static String ipAddress = null;
+
+ public void run(HookContext hookContext) {
+ if (hookContext.getHookType().equals(HookType.PRE_EXEC_HOOK)) {
+ Assert.assertNotNull(hookContext.getIpAddress(), "IP Address is null");
+ ipAddress = hookContext.getIpAddress();
+ Assert.assertNotNull(hookContext.getUserName(), "Username is null");
+ userName = hookContext.getUserName();
+ }
+ }
+ }
+ private static MiniHS2 miniHS2 = null;
+ private static MiniHiveKdc miniHiveKdc = null;
+ private static Map<String, String> confOverlay = new HashMap<String, String>();
+ private Connection hs2Conn;
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ Class.forName(MiniHS2.getJdbcDriverName());
+ confOverlay.put(ConfVars.POSTEXECHOOKS.varname, PostExecHook.class.getName());
+ confOverlay.put(ConfVars.PREEXECHOOKS.varname, PreExecHook.class.getName());
+
+ HiveConf hiveConf = new HiveConf();
+ miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf);
+ miniHS2 = MiniHiveKdc.getMiniHS2WithKerb(miniHiveKdc, hiveConf);
+ miniHS2.start(confOverlay);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (hs2Conn != null) {
+ try {
+ hs2Conn.close();
+ } catch (Exception e) {
+ // Ignore shutdown errors since there are negative tests
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ miniHS2.stop();
+ }
+
+ /**
+ * Test get IpAddress and username from hook.
+ * @throws Exception
+ */
+ @Test
+ public void testIpUserName() throws Exception {
+ miniHiveKdc.loginUser(MiniHiveKdc.HIVE_TEST_USER_1);
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL());
+
+ Statement stmt = hs2Conn.createStatement();
+ stmt.executeQuery("show tables");
+
+ Assert.assertEquals(MiniHiveKdc.HIVE_TEST_USER_1, PostExecHook.userName);
+ Assert.assertNotNull(PostExecHook.ipAddress);
+ Assert.assertTrue(PostExecHook.ipAddress.contains("127.0.0.1"));
+
+ Assert.assertEquals(MiniHiveKdc.HIVE_TEST_USER_1, PreExecHook.userName);
+ Assert.assertNotNull(PreExecHook.ipAddress);
+ Assert.assertTrue(PreExecHook.ipAddress.contains("127.0.0.1"));
+ }
+}
Added: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java?rev=1615022&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java (added)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java Fri Aug 1 01:59:52 2014
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//The tests here are heavily based on some timing, so there is some chance to fail.
+package org.apache.hadoop.hive.hooks;
+
+import java.util.Properties;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext.HookType;
+import org.apache.hive.jdbc.HiveConnection;
+import org.apache.hive.service.server.HiveServer2;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests information retrieved from hooks.
+ */
+public class TestHs2Hooks {
+
+ private static HiveServer2 hiveServer2;
+
+ public static class PreExecHook implements ExecuteWithHookContext {
+ public static String userName = null;
+ public static String ipAddress = null;
+
+ public void run(HookContext hookContext) {
+ if (hookContext.getHookType().equals(HookType.PRE_EXEC_HOOK)) {
+ Assert.assertNotNull(hookContext.getIpAddress(), "IP Address is null");
+ ipAddress = hookContext.getIpAddress();
+ Assert.assertNotNull(hookContext.getUserName(), "Username is null");
+ userName = hookContext.getUserName();
+ }
+ }
+ }
+
+ public static class PostExecHook implements ExecuteWithHookContext {
+ public static String userName = null;
+ public static String ipAddress = null;
+
+ public void run(HookContext hookContext) {
+ if (hookContext.getHookType().equals(HookType.POST_EXEC_HOOK)) {
+ Assert.assertNotNull(hookContext.getIpAddress(), "IP Address is null");
+ ipAddress = hookContext.getIpAddress();
+ Assert.assertNotNull(hookContext.getUserName(), "Username is null");
+ userName = hookContext.getUserName();
+ }
+ }
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS,
+ PreExecHook.class.getName());
+ hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS,
+ PostExecHook.class.getName());
+
+ hiveServer2 = new HiveServer2();
+ hiveServer2.init(hiveConf);
+ hiveServer2.start();
+ Thread.sleep(2000);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ if (hiveServer2 != null) {
+ hiveServer2.stop();
+ }
+ }
+
+ /**
+ * Test get IpAddress and username from hook.
+ * @throws Exception
+ */
+ @Test
+ public void testIpUserName() throws Exception {
+ Properties connProp = new Properties();
+ connProp.setProperty("user", System.getProperty("user.name"));
+ connProp.setProperty("password", "");
+ HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:10000/default", connProp);
+
+ connection.createStatement().execute("show tables");
+
+ Assert.assertEquals(System.getProperty("user.name"), PostExecHook.userName);
+ Assert.assertNotNull(PostExecHook.ipAddress);
+ Assert.assertTrue(PostExecHook.ipAddress.contains("127.0.0.1"));
+
+ Assert.assertEquals(System.getProperty("user.name"), PreExecHook.userName);
+ Assert.assertNotNull(PreExecHook.ipAddress);
+ Assert.assertTrue(PreExecHook.ipAddress.contains("127.0.0.1"));
+
+ connection.close();
+ }
+}
+
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1615022&r1=1615021&r2=1615022&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Aug 1 01:59:52 2014
@@ -1252,7 +1252,8 @@ public class Driver implements CommandPr
}
resStream = null;
- HookContext hookContext = new HookContext(plan, conf, ctx.getPathToCS());
+ SessionState ss = SessionState.get();
+ HookContext hookContext = new HookContext(plan, conf, ctx.getPathToCS(), ss.getUserName(), ss.getUserIpAddress());
hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK);
for (Hook peh : getHooks(HiveConf.ConfVars.PREEXECHOOKS)) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java?rev=1615022&r1=1615021&r2=1615022&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java Fri Aug 1 01:59:52 2014
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -51,13 +50,11 @@ public class HookContext {
private UserGroupInformation ugi;
private HookType hookType;
final private Map<String, ContentSummary> inputPathToContentSummary;
-
- public HookContext(QueryPlan queryPlan, HiveConf conf) throws Exception{
- this(queryPlan, conf, new ConcurrentHashMap<String, ContentSummary>());
- }
+ private final String ipAddress;
+ private final String userName;
public HookContext(QueryPlan queryPlan, HiveConf conf,
- Map<String, ContentSummary> inputPathToContentSummary) throws Exception {
+ Map<String, ContentSummary> inputPathToContentSummary, String userName, String ipAddress) throws Exception {
this.queryPlan = queryPlan;
this.conf = conf;
this.inputPathToContentSummary = inputPathToContentSummary;
@@ -69,6 +66,8 @@ public class HookContext {
if(SessionState.get() != null){
linfo = SessionState.get().getLineageState().getLineageInfo();
}
+ this.ipAddress = ipAddress;
+ this.userName = userName;
}
public QueryPlan getQueryPlan() {
@@ -143,7 +142,15 @@ public class HookContext {
this.hookType = hookType;
}
+ public String getIpAddress() {
+ return this.ipAddress;
+ }
+
public String getOperationName() {
return SessionState.get().getHiveOperation().name();
}
+
+ public String getUserName() {
+ return this.userName;
+ }
}
Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java?rev=1615022&r1=1615021&r2=1615022&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/CLIService.java Fri Aug 1 01:59:52 2014
@@ -46,6 +46,7 @@ import org.apache.hive.service.Composite
import org.apache.hive.service.ServiceException;
import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.cli.operation.Operation;
+import org.apache.hive.service.cli.session.HiveSession;
import org.apache.hive.service.cli.session.SessionManager;
import org.apache.hive.service.cli.thrift.TProtocolVersion;
@@ -149,17 +150,41 @@ public class CLIService extends Composit
super.stop();
}
+ /**
+ * @deprecated Use {@link #openSession(TProtocolVersion, String, String, String, Map)}
+ */
+ @Deprecated
public SessionHandle openSession(TProtocolVersion protocol, String username, String password,
Map<String, String> configuration) throws HiveSQLException {
- SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, configuration, false, null);
+ SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration, false, null);
LOG.debug(sessionHandle + ": openSession()");
return sessionHandle;
}
+ /**
+ * @deprecated Use {@link #openSessionWithImpersonation(TProtocolVersion, String, String, String, Map, String)}
+ */
+ @Deprecated
public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
String password, Map<String, String> configuration, String delegationToken)
throws HiveSQLException {
- SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, configuration,
+ SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, null, configuration,
+ true, delegationToken);
+ LOG.debug(sessionHandle + ": openSession()");
+ return sessionHandle;
+ }
+
+ public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
+ Map<String, String> configuration) throws HiveSQLException {
+ SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration, false, null);
+ LOG.debug(sessionHandle + ": openSession()");
+ return sessionHandle;
+ }
+
+ public SessionHandle openSessionWithImpersonation(TProtocolVersion protocol, String username,
+ String password, String ipAddress, Map<String, String> configuration, String delegationToken)
+ throws HiveSQLException {
+ SessionHandle sessionHandle = sessionManager.openSession(protocol, username, password, ipAddress, configuration,
true, delegationToken);
LOG.debug(sessionHandle + ": openSession()");
return sessionHandle;
@@ -171,7 +196,7 @@ public class CLIService extends Composit
@Override
public SessionHandle openSession(String username, String password, Map<String, String> configuration)
throws HiveSQLException {
- SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, configuration, false, null);
+ SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration, false, null);
LOG.debug(sessionHandle + ": openSession()");
return sessionHandle;
}
@@ -182,7 +207,7 @@ public class CLIService extends Composit
@Override
public SessionHandle openSessionWithImpersonation(String username, String password, Map<String, String> configuration,
String delegationToken) throws HiveSQLException {
- SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, configuration,
+ SessionHandle sessionHandle = sessionManager.openSession(SERVER_VERSION, username, password, null, configuration,
true, delegationToken);
LOG.debug(sessionHandle + ": openSession()");
return sessionHandle;
Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1615022&r1=1615021&r2=1615022&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Fri Aug 1 01:59:52 2014
@@ -112,23 +112,22 @@ public class SessionManager extends Comp
}
}
- public SessionHandle openSession(TProtocolVersion protocol, String username, String password,
+ public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
Map<String, String> sessionConf) throws HiveSQLException {
- return openSession(protocol, username, password, sessionConf, false, null);
+ return openSession(protocol, username, password, ipAddress, sessionConf, false, null);
}
- public SessionHandle openSession(TProtocolVersion protocol, String username, String password,
+ public SessionHandle openSession(TProtocolVersion protocol, String username, String password, String ipAddress,
Map<String, String> sessionConf, boolean withImpersonation, String delegationToken)
throws HiveSQLException {
HiveSession session;
if (withImpersonation) {
HiveSessionImplwithUGI hiveSessionUgi = new HiveSessionImplwithUGI(protocol, username, password,
- hiveConf, TSetIpAddressProcessor.getUserIpAddress(), delegationToken);
+ hiveConf, ipAddress, delegationToken);
session = HiveSessionProxy.getProxy(hiveSessionUgi, hiveSessionUgi.getSessionUgi());
hiveSessionUgi.setProxySession(session);
} else {
- session = new HiveSessionImpl(protocol, username, password, hiveConf,
- TSetIpAddressProcessor.getUserIpAddress());
+ session = new HiveSessionImpl(protocol, username, password, hiveConf, ipAddress);
}
session.setSessionManager(this);
session.setOperationManager(operationManager);
Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java?rev=1615022&r1=1615021&r2=1615022&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java Fri Aug 1 01:59:52 2014
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.AbstractService;
import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.auth.TSetIpAddressProcessor;
@@ -256,6 +257,7 @@ public abstract class ThriftCLIService e
SessionHandle getSessionHandle(TOpenSessionReq req, TOpenSessionResp res)
throws HiveSQLException, LoginException, IOException {
String userName = getUserName(req);
+ String ipAddress = getIpAddress();
TProtocolVersion protocol = getMinVersion(CLIService.SERVER_VERSION,
req.getClient_protocol());
SessionHandle sessionHandle;
@@ -263,10 +265,10 @@ public abstract class ThriftCLIService e
(userName != null)) {
String delegationTokenStr = getDelegationToken(userName);
sessionHandle = cliService.openSessionWithImpersonation(protocol, userName,
- req.getPassword(), req.getConfiguration(), delegationTokenStr);
+ req.getPassword(), ipAddress, req.getConfiguration(), delegationTokenStr);
} else {
sessionHandle = cliService.openSession(protocol, userName, req.getPassword(),
- req.getConfiguration());
+ ipAddress, req.getConfiguration());
}
res.setServerProtocolVersion(protocol);
return sessionHandle;