You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/08/29 01:30:22 UTC
svn commit: r1621228 - in /hive/trunk:
itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/
itests/hive-unit/src/test/java/org/apache/hive/service/
metastore/src/java/org/apache/hadoop/hive/metastore/
service/src/java/org/apache/hive/service/cl...
Author: brock
Date: Thu Aug 28 23:30:22 2014
New Revision: 1621228
URL: http://svn.apache.org/r1621228
Log:
HIVE-6245 - HS2 creates DBs/Tables with wrong ownership when HMS setugi is true (Venki Korukanti via Brock)
Added:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2ImpersonationWithRemoteMS.java
Modified:
hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
Modified: hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java?rev=1621228&r1=1621227&r2=1621228&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java (original)
+++ hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java Thu Aug 28 23:30:22 2014
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
@@ -49,6 +50,7 @@ public class MiniHS2 extends AbstractHiv
public static final String HS2_BINARY_MODE = "binary";
public static final String HS2_HTTP_MODE = "http";
private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
+ private static final FsPermission FULL_PERM = new FsPermission((short)00777);
private HiveServer2 hiveServer2 = null;
private final File baseDir;
private final Path baseDfsDir;
@@ -59,6 +61,7 @@ public class MiniHS2 extends AbstractHiv
private boolean useMiniKdc = false;
private final String serverPrincipal;
private final String serverKeytab;
+ private final boolean isMetastoreRemote;
public static class Builder {
private HiveConf hiveConf = new HiveConf();
@@ -67,6 +70,7 @@ public class MiniHS2 extends AbstractHiv
private String serverPrincipal;
private String serverKeytab;
private boolean isHTTPTransMode = false;
+ private boolean isMetastoreRemote;
public Builder() {
}
@@ -83,6 +87,11 @@ public class MiniHS2 extends AbstractHiv
return this;
}
+ public Builder withRemoteMetastore() {
+ this.isMetastoreRemote = true;
+ return this;
+ }
+
public Builder withConf(HiveConf hiveConf) {
this.hiveConf = hiveConf;
return this;
@@ -107,7 +116,8 @@ public class MiniHS2 extends AbstractHiv
} else {
hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE);
}
- return new MiniHS2(hiveConf, useMiniMR, useMiniKdc, serverPrincipal, serverKeytab);
+ return new MiniHS2(hiveConf, useMiniMR, useMiniKdc, serverPrincipal, serverKeytab,
+ isMetastoreRemote);
}
}
@@ -139,12 +149,14 @@ public class MiniHS2 extends AbstractHiv
return useMiniKdc;
}
- private MiniHS2(HiveConf hiveConf, boolean useMiniMR, boolean useMiniKdc, String serverPrincipal, String serverKeytab) throws Exception {
+ private MiniHS2(HiveConf hiveConf, boolean useMiniMR, boolean useMiniKdc,
+ String serverPrincipal, String serverKeytab, boolean isMetastoreRemote) throws Exception {
super(hiveConf, "localhost", MetaStoreUtils.findFreePort(), MetaStoreUtils.findFreePort());
this.useMiniMR = useMiniMR;
this.useMiniKdc = useMiniKdc;
this.serverPrincipal = serverPrincipal;
this.serverKeytab = serverKeytab;
+ this.isMetastoreRemote = isMetastoreRemote;
baseDir = Files.createTempDir();
FileSystem fs;
if (useMiniMR) {
@@ -169,6 +181,9 @@ public class MiniHS2 extends AbstractHiv
fs.mkdirs(baseDfsDir);
Path wareHouseDir = new Path(baseDfsDir, "warehouse");
+ // Create warehouse with 777, so that user impersonation has no issues.
+ FileSystem.mkdirs(fs, wareHouseDir, FULL_PERM);
+
fs.mkdirs(wareHouseDir);
setWareHouseDir(wareHouseDir.toString());
System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL);
@@ -180,10 +195,15 @@ public class MiniHS2 extends AbstractHiv
hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, getHttpPort());
Path scratchDir = new Path(baseDfsDir, "scratch");
- fs.mkdirs(scratchDir);
+
+ // Create scratchdir with 777, so that user impersonation has no issues.
+ FileSystem.mkdirs(fs, scratchDir, FULL_PERM);
System.setProperty(HiveConf.ConfVars.SCRATCHDIR.varname, scratchDir.toString());
- System.setProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.varname,
- baseDir.getPath() + File.separator + "scratch");
+ hiveConf.setVar(ConfVars.SCRATCHDIR, scratchDir.toString());
+
+ String localScratchDir = baseDir.getPath() + File.separator + "scratch";
+ System.setProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.varname, localScratchDir);
+ hiveConf.setVar(ConfVars.LOCALSCRATCHDIR, localScratchDir);
}
public MiniHS2(HiveConf hiveConf) throws Exception {
@@ -191,10 +211,17 @@ public class MiniHS2 extends AbstractHiv
}
public MiniHS2(HiveConf hiveConf, boolean useMiniMR) throws Exception {
- this(hiveConf, useMiniMR, false, null, null);
+ this(hiveConf, useMiniMR, false, null, null, false);
}
public void start(Map<String, String> confOverlay) throws Exception {
+ if (isMetastoreRemote) {
+ int metaStorePort = MetaStoreUtils.findFreePort();
+ getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
+ MetaStoreUtils.startMetaStore(metaStorePort,
+ ShimLoader.getHadoopThriftAuthBridge(), getHiveConf());
+ }
+
hiveServer2 = new HiveServer2();
// Set confOverlay parameters
for (Map.Entry<String, String> entry : confOverlay.entrySet()) {
@@ -208,6 +235,9 @@ public class MiniHS2 extends AbstractHiv
public void stop() {
verifyStarted();
+ // Currently there is no way to stop the MetaStore service. It will be stopped when the
+ // test JVM exits. This is how other tests are also using MetaStore server.
+
hiveServer2.stop();
setStarted(false);
try {
Added: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2ImpersonationWithRemoteMS.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2ImpersonationWithRemoteMS.java?rev=1621228&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2ImpersonationWithRemoteMS.java (added)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2ImpersonationWithRemoteMS.java Thu Aug 28 23:30:22 2014
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.util.HashMap;
+
+/**
+ * Test HiveServer2 sends correct user name to remote MetaStore server for user impersonation.
+ */
+public class TestHS2ImpersonationWithRemoteMS {
+
+ private static MiniHS2 miniHS2 = null;
+
+ @BeforeClass
+ public static void startServices() throws Exception {
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS, 1);
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS, 1);
+ hiveConf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI, true);
+ hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+
+ miniHS2 = new MiniHS2.Builder()
+ .withMiniMR()
+ .withRemoteMetastore()
+ .withConf(hiveConf).build();
+
+ miniHS2.start(new HashMap<String, String>());
+ }
+
+ @AfterClass
+ public static void stopServices() throws Exception {
+ if (miniHS2 != null && miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ }
+
+ @Test
+ public void testImpersonation() throws Exception {
+ assertTrue("Test setup failed. MiniHS2 is not initialized",
+ miniHS2 != null && miniHS2.isStarted());
+
+ Class.forName(MiniHS2.getJdbcDriverName());
+
+ // Create two tables one as user "foo" and other as user "bar"
+ Connection hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "foo", null);
+ Statement stmt = hs2Conn.createStatement();
+
+ String tableName = "foo_table";
+ stmt.execute("drop table if exists " + tableName);
+ stmt.execute("create table " + tableName + " (value string)");
+
+ stmt.close();
+ hs2Conn.close();
+
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "bar", null);
+ stmt = hs2Conn.createStatement();
+
+ tableName = "bar_table";
+ stmt.execute("drop table if exists " + tableName);
+ stmt.execute("create table " + tableName + " (value string)");
+
+ stmt.close();
+ hs2Conn.close();
+
+ MiniDFSShim dfs = miniHS2.getDfs();
+ FileSystem fs = dfs.getFileSystem();
+
+ FileStatus[] files = fs.listStatus(miniHS2.getWareHouseDir());
+ boolean fooTableValidated = false;
+ boolean barTableValidated = false;
+ for(FileStatus file : files) {
+ final String name = file.getPath().getName();
+ final String owner = file.getOwner();
+ if (name.equals("foo_table")) {
+ fooTableValidated = owner.equals("foo");
+ assertTrue(String.format("User 'foo' table has wrong ownership '%s'", owner),
+ fooTableValidated);
+ } else if (name.equals("bar_table")) {
+ barTableValidated = owner.equals("bar");
+ assertTrue(String.format("User 'bar' table has wrong ownership '%s'", owner),
+ barTableValidated);
+ } else {
+ fail(String.format("Unexpected table directory '%s' in warehouse", name));
+ }
+
+ System.out.println(String.format("File: %s, Owner: %s", name, owner));
+ }
+
+ assertTrue("User 'foo' table not found in warehouse", fooTableValidated);
+ assertTrue("User 'bar' table not found in warehouse", barTableValidated);
+ }
+}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1621228&r1=1621227&r2=1621228&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Thu Aug 28 23:30:22 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
@@ -1043,11 +1044,17 @@ public class MetaStoreUtils {
public static void startMetaStore(final int port,
final HadoopThriftAuthBridge bridge) throws Exception {
+ startMetaStore(port, bridge, new HiveConf(HMSHandler.class));
+ }
+
+ public static void startMetaStore(final int port,
+ final HadoopThriftAuthBridge bridge, final HiveConf hiveConf)
+ throws Exception{
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
- HiveMetaStore.startMetaStore(port, bridge);
+ HiveMetaStore.startMetaStore(port, bridge, hiveConf);
} catch (Throwable e) {
LOG.error("Metastore Thrift Server threw an exception...",e);
}
@@ -1057,6 +1064,7 @@ public class MetaStoreUtils {
thread.start();
loopUntilHMSReady(port);
}
+
/**
* A simple connect test to make sure that the metastore is up
* @throws Exception
Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java?rev=1621228&r1=1621227&r2=1621228&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java Thu Aug 28 23:30:22 2014
@@ -48,6 +48,14 @@ public class HiveSessionImplwithUGI exte
super(protocol, username, password, hiveConf, ipAddress);
setSessionUGI(username);
setDelegationToken(delegationToken);
+
+ // create a new metastore connection for this particular user session
+ Hive.set(null);
+ try {
+ sessionHive = Hive.get(getHiveConf());
+ } catch (HiveException e) {
+ throw new HiveSQLException("Failed to setup metastore connection", e);
+ }
}
// setup appropriate UGI for the session
@@ -115,13 +123,6 @@ public class HiveSessionImplwithUGI exte
} catch (IOException e) {
throw new HiveSQLException("Couldn't setup delegation token in the ugi", e);
}
- // create a new metastore connection using the delegation token
- Hive.set(null);
- try {
- sessionHive = Hive.get(getHiveConf());
- } catch (HiveException e) {
- throw new HiveSQLException("Failed to setup metastore connection", e);
- }
}
}