You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/03/25 16:12:02 UTC
svn commit: r1305041 - in /hive/trunk:
metastore/src/java/org/apache/hadoop/hive/metastore/
metastore/src/test/org/apache/hadoop/hive/metastore/
shims/src/common/java/org/apache/hadoop/hive/thrift/
Author: hashutosh
Date: Sun Mar 25 14:12:02 2012
New Revision: 1305041
URL: http://svn.apache.org/viewvc?rev=1305041&view=rev
Log:
HIVE-2797: Make the IP address of a Thrift client available to HMSHandler. (Kevin Wilfong via Ashutosh Chauhan)
Added:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/IpAddressListener.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1305041&r1=1305040&r2=1305041&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Sun Mar 25 14:12:02 2012
@@ -203,6 +203,26 @@ public class HiveMetaStore extends Thrif
}
};
+ // This will only be set if the metastore is being accessed from a metastore Thrift server,
+ // not if it is from the CLI. Also, only if the TTransport being used to connect is an
+ // instance of TSocket.
+ private static ThreadLocal<String> threadLocalIpAddress = new ThreadLocal<String>() {
+ @Override
+ protected synchronized String initialValue() {
+ return null;
+ }
+ };
+
+ public static void setIpAddress(String ipAddress) {
+ threadLocalIpAddress.set(ipAddress);
+ }
+
+ // This will return null if the metastore is not being accessed from a metastore Thrift server,
+ // or if the TTransport being used to connect is not an instance of TSocket.
+ public static String getIpAddress() {
+ return threadLocalIpAddress.get();
+ }
+
public static Integer get() {
return threadLocalId.get();
}
@@ -2914,7 +2934,7 @@ public class HiveMetaStore extends Thrif
LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
} else {
transFactory = new TTransportFactory();
- processor = new ThriftHiveMetastore.Processor<HMSHandler>(handler);
+ processor = new TSetIpAddressProcessor<HMSHandler>(handler);
LOG.info("Starting DB backed MetaStore Server");
}
}
Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java?rev=1305041&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java Sun Mar 25 14:12:02 2012
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.lang.reflect.InvocationTargetException;
+import java.net.Socket;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
+import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+
+/**
+ * TSetIpAddressProcessor passes the IP address of the Thrift client to the HMSHandler.
+ */
+public class TSetIpAddressProcessor<I extends Iface> extends ThriftHiveMetastore.Processor<Iface> {
+
+ @SuppressWarnings("unchecked")
+ public TSetIpAddressProcessor(I iface) throws SecurityException, NoSuchFieldException,
+ IllegalArgumentException, IllegalAccessException, NoSuchMethodException,
+ InvocationTargetException {
+ super(iface);
+ }
+
+ @Override
+ public boolean process(final TProtocol in, final TProtocol out) throws TException {
+ setIpAddress(in);
+
+ return super.process(in, out);
+ }
+
+ protected void setIpAddress(final TProtocol in) {
+ TTransport transport = in.getTransport();
+ if (!(transport instanceof TSocket)) {
+ return;
+ }
+ setIpAddress(((TSocket)transport).getSocket());
+ }
+
+ protected void setIpAddress(final Socket inSocket) {
+ HMSHandler.setIpAddress(inSocket.getInetAddress().toString());
+ }
+}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java?rev=1305041&r1=1305040&r2=1305041&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java Sun Mar 25 14:12:02 2012
@@ -21,12 +21,12 @@ package org.apache.hadoop.hive.metastore
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.net.Socket;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Processor;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.set_ugi_args;
@@ -53,7 +53,7 @@ import org.apache.thrift.protocol.TType;
* Note that old clients will never call set_ugi() and thus ugi will never be received on server
* side, in which case server exhibits previous behavior and continues as usual.
*/
-public class TUGIBasedProcessor<I extends Iface> extends ThriftHiveMetastore.Processor<Iface> {
+public class TUGIBasedProcessor<I extends Iface> extends TSetIpAddressProcessor<Iface> {
private final I iface;
private final Map<String, org.apache.thrift.ProcessFunction<I, ? extends TBase<?,?>>>
@@ -76,6 +76,7 @@ public class TUGIBasedProcessor<I extend
@Override
public boolean process(final TProtocol in, final TProtocol out) throws TException {
+ setIpAddress(in);
final TMessage msg = in.readMessageBegin();
final ProcessFunction<I, ? extends TBase<?,?>> fn = functions.get(msg.name);
@@ -175,4 +176,13 @@ public class TUGIBasedProcessor<I extend
oprot.writeMessageEnd();
oprot.getTransport().flush();
}
+
+ @Override
+ protected void setIpAddress(final TProtocol in) {
+ TUGIContainingTransport ugiTrans = (TUGIContainingTransport)in.getTransport();
+ Socket socket = ugiTrans.getSocket();
+ if (socket != null) {
+ setIpAddress(socket);
+ }
+ }
}
Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/IpAddressListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/IpAddressListener.java?rev=1305041&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/IpAddressListener.java (added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/IpAddressListener.java Sun Mar 25 14:12:02 2012
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
+
+/** An implementation for MetaStoreEventListener which checks that the IP Address stored in
+ * HMSHandler matches that of local host, for testing purposes.
+ */
+public class IpAddressListener extends MetaStoreEventListener{
+
+ private static final String LOCAL_HOST = "localhost";
+
+ public IpAddressListener(Configuration config) {
+ super(config);
+ }
+
+ private String getIpFromInetAddress(String addr) {
+ return addr.substring(addr.indexOf('/') + 1);
+ }
+
+ private void checkIpAddress() {
+ try {
+ String localhostIp = InetAddress.getByName(LOCAL_HOST).toString();
+ Assert.assertEquals(getIpFromInetAddress(localhostIp),
+ getIpFromInetAddress(HMSHandler.getIpAddress()));
+ } catch (UnknownHostException e) {
+ Assert.assertTrue("InetAddress.getLocalHost threw an exception: " + e.getMessage(), false);
+ }
+ }
+
+ @Override
+ public void onAddPartition(AddPartitionEvent partition) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onCreateDatabase(CreateDatabaseEvent db) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onCreateTable(CreateTableEvent table) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onDropDatabase(DropDatabaseEvent db) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partition) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent table) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onAlterTable(AlterTableEvent event) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onAlterPartition(AlterPartitionEvent event) throws MetaException {
+ checkIpAddress();
+ }
+
+ @Override
+ public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException {
+ checkIpAddress();
+ }
+}
Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java?rev=1305041&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java (added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java Sun Mar 25 14:12:02 2012
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ *
+ * TestRemoteHiveMetaStoreIpAddress.
+ *
+ * Test which checks that the remote Hive metastore stores the proper IP address using
+ * IpAddressListener
+ */
+public class TestRemoteHiveMetaStoreIpAddress extends TestCase {
+ protected static final String METASTORE_PORT = "39083";
+ private static boolean isServerStarted = false;
+ private static HiveConf hiveConf;
+ private static HiveMetaStoreClient msc;
+
+ private static class RunMS implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+ System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname,
+ IpAddressListener.class.getName());
+ HiveMetaStore.main(new String[] { METASTORE_PORT });
+ } catch (Throwable e) {
+ e.printStackTrace(System.err);
+ assert false;
+ }
+ }
+
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ hiveConf = new HiveConf(this.getClass());
+
+ if (isServerStarted) {
+ assertNotNull("Unable to connect to the MetaStore server", msc);
+ return;
+ }
+
+ System.out.println("Starting MetaStore Server on port " + METASTORE_PORT);
+ Thread t = new Thread(new RunMS());
+ t.start();
+ isServerStarted = true;
+
+ // Wait a little bit for the metastore to start. Should probably have
+ // a better way of detecting if the metastore has started?
+ Thread.sleep(5000);
+ // This is default case with setugi off for both client and server
+ createClient();
+ }
+
+ public void testIpAddress() throws Exception {
+ try {
+
+ Database db = new Database();
+ db.setName("testIpAddressIp");
+ msc.createDatabase(db);
+ msc.dropDatabase(db.getName());
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testIpAddress() failed.");
+ throw e;
+ }
+ }
+
+ protected void createClient() throws Exception {
+ hiveConf.setBoolVar(ConfVars.METASTORE_MODE, false);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + METASTORE_PORT);
+ msc = new HiveMetaStoreClient(hiveConf);
+ }
+}
Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java?rev=1305041&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java (added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java Sun Mar 25 14:12:02 2012
@@ -0,0 +1,11 @@
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+public class TestRemoteUGIHiveMetaStoreIpAddress extends TestRemoteHiveMetaStoreIpAddress {
+ public TestRemoteUGIHiveMetaStoreIpAddress() {
+ super();
+ System.setProperty(ConfVars.METASTORE_EXECUTE_SET_UGI.varname, "true");
+ }
+
+}
Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java?rev=1305041&r1=1305040&r2=1305041&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java Sun Mar 25 14:12:02 2012
@@ -18,9 +18,11 @@
package org.apache.hadoop.hive.thrift;
+import java.net.Socket;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportFactory;
@@ -46,6 +48,18 @@ public class TUGIContainingTransport ext
this.ugi = ugi;
}
+ /**
+ * If the underlying TTransport is an instance of TSocket, it returns the Socket object
+ * which it contains. Otherwise it returns null.
+ */
+ public Socket getSocket() {
+ if (wrapped instanceof TSocket) {
+ return (((TSocket)wrapped).getSocket());
+ }
+
+ return null;
+ }
+
/** Factory to create TUGIContainingTransport.
*/