You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2012/01/19 10:19:39 UTC
svn commit: r1233260 [2/2] - in /hive/branches/branch-0.8-r2:
common/src/java/org/apache/hadoop/hive/conf/ conf/ metastore/if/
metastore/src/gen/thrift/gen-cpp/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/g...
Modified: hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Jan 19 09:19:38 2012
@@ -104,6 +104,7 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.thrift.TUGIContainingTransport;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
@@ -3581,6 +3582,13 @@ public class HiveMetaStore extends Thrif
return ret;
}
+
+ @Override
+ public List<String> set_ugi(String username, List<String> groupNames) throws MetaException,
+ TException {
+ Collections.addAll(groupNames, username);
+ return groupNames;
+ }
}
@@ -3742,19 +3750,28 @@ public class HiveMetaStore extends Thrif
TProcessor processor;
TTransportFactory transFactory;
if (useSasl) {
+ // we are in secure mode.
saslServer = bridge.createServer(
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE),
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL));
-
// start delegation token manager
saslServer.startDelegationTokenSecretManager(conf);
transFactory = saslServer.createTransportFactory();
- processor = saslServer.wrapProcessor(new ThriftHiveMetastore.Processor(
+ processor = saslServer.wrapProcessor(new ThriftHiveMetastore.Processor<HMSHandler>(
new HMSHandler("new db based metaserver", conf)));
+ LOG.info("Starting DB backed MetaStore Server in Secure Mode");
} else {
- processor = new ThriftHiveMetastore.Processor(
- new HMSHandler("new db based metaserver", conf));
+ // we are in unsecure mode.
+ HMSHandler handler = new HMSHandler("new db based metaserver", conf);
+ if (conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)){
+ transFactory = new TUGIContainingTransport.Factory();
+ processor = new TUGIBasedProcessor<HMSHandler>(handler);
+ LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
+ } else{
transFactory = new TTransportFactory();
+ processor = new ThriftHiveMetastore.Processor<HMSHandler>(handler);
+ LOG.info("Starting DB backed MetaStore Server");
+ }
}
TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport)
Modified: hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Thu Jan 19 09:19:38 2012
@@ -28,10 +28,13 @@ import java.lang.reflect.Proxy;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import javax.security.auth.login.LoginException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -60,8 +63,10 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
@@ -219,7 +224,8 @@ public class HiveMetaStoreClient impleme
((TSocket)transport).setTimeout(1000 * conf.getIntVar(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT));
// Wrap thrift connection with SASL if enabled.
- boolean useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL);
+ HadoopShims shim = ShimLoader.getHadoopShims();
+ boolean useSasl = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL);
if (useSasl) {
try {
HadoopThriftAuthBridge.Client authBridge =
@@ -232,7 +238,7 @@ public class HiveMetaStoreClient impleme
// submission.
String tokenSig = conf.get("hive.metastore.token.signature");
// tokenSig could be null
- tokenStrForm = ShimLoader.getHadoopShims().getTokenStrForm(tokenSig);
+ tokenStrForm = shim.getTokenStrForm(tokenSig);
if(tokenStrForm != null) {
// authenticate using delegation tokens via the "DIGEST" mechanism
@@ -264,6 +270,21 @@ public class HiveMetaStoreClient impleme
LOG.warn("Failed to connect to the MetaStore Server...");
}
}
+ if (!useSasl && conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)){
+ // Call set_ugi, only in unsecure mode.
+ try {
+ UserGroupInformation ugi = shim.getUGIForConf(conf);
+ client.set_ugi(ugi.getUserName(), Arrays.asList(ugi.getGroupNames()));
+ } catch (LoginException e) {
+ LOG.warn("Failed to do login. set_ugi() is not successful, Continuing without it.", e);
+ } catch (IOException e) {
+ LOG.warn("Failed to find ugi of client set_ugi() is not successful, " +
+ "Continuing without it.", e);
+ } catch (TException e) {
+ LOG.warn("set_ugi() not successful, Likely cause: new client talking to old server. " +
+ "Continuing without it.", e);
+ }
+ }
}
if (!isConnected) {
throw new MetaException("Could not connect to the MetaStore server!");
Modified: hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java (original)
+++ hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java Thu Jan 19 09:19:38 2012
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.conf.HiveC
public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
- private static final String METASTORE_PORT = "29083";
+ protected static final String METASTORE_PORT = "29083";
private static boolean isServerStarted = false;
public TestRemoteHiveMetaStore() {
@@ -62,14 +62,14 @@ public class TestRemoteHiveMetaStore ext
// Wait a little bit for the metastore to start. Should probably have
// a better way of detecting if the metastore has started?
Thread.sleep(5000);
+ // This is default case with setugi off for both client and server
+ createClient(false);
+ }
- // hive.metastore.local should be defined in HiveConf
+ protected void createClient(boolean setugi) throws Exception {
hiveConf.setBoolVar(ConfVars.METASTORE_MODE, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + METASTORE_PORT);
- hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
- hiveConf.setIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, 60);
-
+ hiveConf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI,setugi);
client = new HiveMetaStoreClient(hiveConf);
}
-
}
Modified: hive/branches/branch-0.8-r2/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/ivy.xml?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/ivy.xml (original)
+++ hive/branches/branch-0.8-r2/shims/ivy.xml Thu Jan 19 09:19:38 2012
@@ -43,6 +43,8 @@
transitive="false"/>
<dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"
transitive="false"/>
+ <dependency org="com.google.guava" name="guava" rev="${guava.version}"
+ transitive="false"/>
<conflict manager="all" />
</dependencies>
</ivy-module>
Modified: hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Thu Jan 19 09:19:38 2012
@@ -21,9 +21,12 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.lang.reflect.Constructor;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
+import javax.security.auth.Subject;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +35,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.ClusterStatus;
@@ -54,6 +56,7 @@ import org.apache.hadoop.mapred.lib.Comb
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UnixUserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.HadoopArchives;
@@ -510,6 +513,21 @@ public class Hadoop20Shims implements Ha
}
@Override
+ public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea) throws
+ IOException, InterruptedException {
+ try {
+ Subject.doAs(SecurityUtil.getSubject(ugi),pvea);
+ } catch (PrivilegedActionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public UserGroupInformation createRemoteUser(String userName, List<String> groupNames) {
+ return new UnixUserGroupInformation(userName, groupNames.toArray(new String[0]));
+ }
+
+ @Override
public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception {
JobTrackerState state;
switch (clusterStatus.getJobTrackerState()) {
Modified: hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Thu Jan 19 09:19:38 2012
@@ -21,6 +21,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.lang.reflect.Constructor;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@@ -30,14 +31,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
-import org.apache.hadoop.hive.shims.Hadoop20Shims;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.Hadoop20Shims.InputSplitShim;
import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
@@ -517,6 +513,16 @@ public class Hadoop20SShims implements H
}
@Override
+ public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea) throws IOException, InterruptedException {
+ ugi.doAs(pvea);
+ }
+
+ @Override
+ public UserGroupInformation createRemoteUser(String userName, List<String> groupNames) {
+ return UserGroupInformation.createRemoteUser(userName);
+ }
+
+ @Override
public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception {
JobTrackerState state;
switch (clusterStatus.getJobTrackerState()) {
Modified: hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java (original)
+++ hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java Thu Jan 19 09:19:38 2012
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUti
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
@@ -189,131 +190,7 @@ import org.apache.thrift.transport.TTran
return new String(Base64.encodeBase64(password)).toCharArray();
}
}
- /**
- * The Thrift SASL transports call Sasl.createSaslServer and Sasl.createSaslClient
- * inside open(). So, we need to assume the correct UGI when the transport is opened
- * so that the SASL mechanisms have access to the right principal. This transport
- * wraps the Sasl transports to set up the right UGI context for open().
- *
- * This is used on the client side, where the API explicitly opens a transport to
- * the server.
- */
- private static class TUGIAssumingTransport extends TFilterTransport {
- private final UserGroupInformation ugi;
-
- public TUGIAssumingTransport(TTransport wrapped, UserGroupInformation ugi) {
- super(wrapped);
- this.ugi = ugi;
- }
-
- @Override
- public void open() throws TTransportException {
- try {
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
- public Void run() {
- try {
- wrapped.open();
- } catch (TTransportException tte) {
- // Wrap the transport exception in an RTE, since UGI.doAs() then goes
- // and unwraps this for us out of the doAs block. We then unwrap one
- // more time in our catch clause to get back the TTE. (ugh)
- throw new RuntimeException(tte);
- }
- return null;
- }
- });
- } catch (IOException ioe) {
- assert false : "Never thrown!";
- throw new RuntimeException("Received an ioe we never threw!", ioe);
- } catch (InterruptedException ie) {
- assert false : "We never expect to see an InterruptedException thrown in this block";
- throw new RuntimeException("Received an ie we never threw!", ie);
- } catch (RuntimeException rte) {
- if (rte.getCause() instanceof TTransportException) {
- throw (TTransportException)rte.getCause();
- } else {
- throw rte;
- }
- }
}
- }
- /**
- * Transport that simply wraps another transport.
- * This is the equivalent of FilterInputStream for Thrift transports.
- */
- private static class TFilterTransport extends TTransport {
- protected final TTransport wrapped;
-
- public TFilterTransport(TTransport wrapped) {
- this.wrapped = wrapped;
- }
-
- @Override
- public void open() throws TTransportException {
- wrapped.open();
- }
-
- @Override
- public boolean isOpen() {
- return wrapped.isOpen();
- }
-
- @Override
- public boolean peek() {
- return wrapped.peek();
- }
-
- @Override
- public void close() {
- wrapped.close();
- }
-
- @Override
- public int read(byte[] buf, int off, int len) throws TTransportException {
- return wrapped.read(buf, off, len);
- }
-
- @Override
- public int readAll(byte[] buf, int off, int len) throws TTransportException {
- return wrapped.readAll(buf, off, len);
- }
-
- @Override
- public void write(byte[] buf) throws TTransportException {
- wrapped.write(buf);
- }
-
- @Override
- public void write(byte[] buf, int off, int len) throws TTransportException {
- wrapped.write(buf, off, len);
- }
-
- @Override
- public void flush() throws TTransportException {
- wrapped.flush();
- }
-
- @Override
- public byte[] getBuffer() {
- return wrapped.getBuffer();
- }
-
- @Override
- public int getBufferPosition() {
- return wrapped.getBufferPosition();
- }
-
- @Override
- public int getBytesRemainingInBuffer() {
- return wrapped.getBytesRemainingInBuffer();
- }
-
- @Override
- public void consumeBuffer(int len) {
- wrapped.consumeBuffer(len);
- }
- }
- }
public static class Server extends HadoopThriftAuthBridge.Server {
final UserGroupInformation realUgi;
Modified: hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Thu Jan 19 09:19:38 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.shims;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
import javax.security.auth.login.LoginException;
@@ -164,6 +166,24 @@ public interface HadoopShims {
public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException, IOException;
/**
+ * Used by metastore server to perform requested rpc in client context.
+ * @param ugi
+ * @param pvea
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea) throws
+ IOException, InterruptedException;
+
+ /**
+ * Used by metastore server to creates UGI object for a remote user.
+ * @param userName remote User Name
+ * @param groupNames group names associated with remote user name
+ * @return UGI created for the remote user.
+ */
+
+ public UserGroupInformation createRemoteUser(String userName, List<String> groupNames);
+ /**
* Get the short name corresponding to the subject in the passed UGI
*
* In secure versions of Hadoop, this returns the short name (after