You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/08/30 08:59:09 UTC
svn commit: r1518894 - in /hive/trunk/hcatalog:
src/docs/src/documentation/content/xdocs/ src/test/e2e/templeton/
src/test/e2e/templeton/drivers/ webhcat/svr/src/main/config/
webhcat/svr/src/main/java/org/apache/hcatalog/templeton/
webhcat/svr/src/main...
Author: thejas
Date: Fri Aug 30 06:59:08 2013
New Revision: 1518894
URL: http://svn.apache.org/r1518894
Log:
HIVE-4601 : WebHCat needs to support proxy users (Eugene Koifman via Thejas Nair)
Modified:
hive/trunk/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
hive/trunk/hcatalog/src/test/e2e/templeton/README.txt
hive/trunk/hcatalog/src/test/e2e/templeton/build.xml
hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java
Modified: hive/trunk/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml (original)
+++ hive/trunk/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml Fri Aug 30 06:59:08 2013
@@ -286,6 +286,46 @@ Using localhost in metastore uri does no
principal.</td>
</tr>
+ <tr>
+ <td><strong>webhcat.proxyuser.#USER#.hosts</strong></td>
+ <td>None</td>
+ <td>List of client hosts from which the '#USER#' user is allowed to perform
+ 'doAs' operations.
+
+ The '#USER#' must be replaced with the username of the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard, which means every host is allowed,
+ or a comma-separated list of hostnames.
+
+ If value is a blank string or webhcat.proxyuser.#USER#.hosts is missing,
+ no hosts will be allowed.
+
+ For multiple users copy this property and replace the user name
+ in the property name.</td>
+ </tr>
+ <tr>
+ <td><strong>webhcat.proxyuser.#USER#.groups</strong></td>
+ <td>None</td>
+ <td>List of groups the '#USER#' user is allowed to impersonate users
+ from to perform 'doAs' operations.
+
+ The '#USER#' must be replaced with the username of the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard, which means any doAs value is
+ allowed, or a comma-separated list of groups.
+
+ If value is an empty list or webhcat.proxyuser.#USER#.groups is missing,
+ every doAs call value will fail.
+
+ For multiple users copy this property and replace the user name
+ in the property name.
+
+ The username->usergroup mapping is performed using Hadoop API which is
+ controlled by hadoop.security.group.mapping property.</td>
+ </tr>
+
</table>
</section>
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/README.txt
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/README.txt?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/README.txt (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/README.txt Fri Aug 30 06:59:08 2013
@@ -123,6 +123,29 @@ ant test-hcat-authorization -Dkeytab.dir
The <keytab files dir> is expected to have keytab filenames of the form - user_name.*keytab .
+Running WebHCat doas tests
+--------------------------
+ant clean test-doas -Dinpdir.hdfs=/user/ekoifman/webhcate2e -Dsecure.mode=no
+ -Dharness.webhdfs.url=http://localhost:8085 -Dharness.templeton.url=http://localhost:50111
+ -Dtests.to.run='-t doAsTests' -Dtest.user.name=hue -Ddoas.user=joe
+
+The canonical example, is WebHCat server is running as user 'hcat', end user 'joe' is using Hue,
+which generates a request to WebHCat. If Hue specifies doAs=joe, then the commands that WebHCat
+submits to Hadoop will be run as user 'joe'.
+
+In order for this test suite to work, webhcat-site.xml should have webhcat.proxyuser.hue.groups
+and webhcat.proxyuser.hue.hosts defined, i.e. 'hue' should be allowed to impersonate 'joe'.
+[Of course, 'hcat' proxyuser should be configured in core-site.xml for the command to succeed.]
+
+Furthermore, metastore side file based security should be enabled. To do this 3 properties in
+hive-site.xml should be configured:
+1) hive.security.metastore.authorization.manager set to
+ org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider
+2) hive.security.metastore.authenticator.manager set to
+ org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator
+3) hive.metastore.pre.event.listeners set to
+ org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener
+4) hive.metastore.execute.setugi set to true
Notes
-----
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/build.xml?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/build.xml (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/build.xml Fri Aug 30 06:59:08 2013
@@ -83,11 +83,15 @@
<target name="test" depends="property-check, init-test">
<!-- fork (parallelization) factors for e2e tests execution.
- Defaults are 1, which means *no* parellelization: -->
- <property name="fork.factor.group" value="1"/>
- <property name="fork.factor.conf.file" value="1"/>
+ Defaults are 1, which means *no* parellelization:
+ if group=3, then 3 .conf files will be processed in parallel
+ if conf.file=2 there will be 2 thread per .conf file, each thread
+ executing a single group (identified by 'name' element) -->
+ <property name="fork.factor.group" value="5"/>
+ <property name="fork.factor.conf.file" value="5"/>
<property name="e2e.debug" value="false"/>
<property name="tests.to.run" value=""/>
+
<exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="."/>
<env key="TH_WORKING_DIR" value="${test.location}"/>
@@ -141,6 +145,42 @@
<arg value="${basedir}/tests/hcatperms.conf"/>
</exec>
</target>
+ <target name="test-doas" depends="property-check, init-test" description="See README.txt for instructions">
+ <!-- fork (parallelization) factors for e2e tests execution.
+ Defaults are 1, which means *no* parellelization:
+ if group=3, then 3 .conf files will be processed in parallel
+ if conf.file=2 there will be 2 thread per .conf file, each thread
+ executing a single group (identified by 'name' element) -->
+ <property name="fork.factor.group" value="1"/>
+ <property name="fork.factor.conf.file" value="1"/>
+ <property name="e2e.debug" value="false"/>
+ <property name="tests.to.run" value=""/>
+ <property name="doas.user.tmp" value="${doas.user}" />
+ <condition property="doas.user" value="${test.user.name}">
+ <!--default doas.user (if not set) to test.user.name-->
+ <isset property="doas.user.tmp"/>
+ </condition>
+
+ <exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="."/>
+ <env key="TH_WORKING_DIR" value="${test.location}"/>
+ <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
+ <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+ <env key="TH_OUT" value="."/>
+ <env key="TH_ROOT" value="."/>
+ <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>
+ <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
+ <env key="E2E_DEBUG" value="${e2e.debug}"/>
+ <env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
+ <env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
+ <env key="USER_NAME" value="${test.user.name}"/>
+ <env key="DOAS_USER" value="${doas.user}"/>
+ <env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
+ <env key="SECURE_MODE" value="${secure.mode}"/>
+ <arg line="${tests.to.run}"/>
+ <arg value="${basedir}/tests/doas.conf"/>
+ </exec>
+ </target>
<target name="clean">
<delete dir="${test.location}"/>
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Fri Aug 30 06:59:08 2013
@@ -174,6 +174,7 @@ sub globalSetup
$globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'};
$globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'};
$globalHash->{'current_user'} = $ENV{'USER_NAME'};
+ $globalHash->{'DOAS_USER'} = $ENV{'DOAS_USER'};
$globalHash->{'current_group_user'} = $ENV{'GROUP_USER_NAME'};
$globalHash->{'current_other_user'} = $ENV{'OTHER_USER_NAME'};
$globalHash->{'current_group'} = $ENV{'GROUP_NAME'};
@@ -317,6 +318,7 @@ sub replaceParametersInArg
}
my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'};
$arg =~ s/:UNAME:/$testCmd->{'current_user'}/g;
+ $arg =~ s/:DOAS:/$testCmd->{'DOAS_USER'}/g;
$arg =~ s/:UNAME_GROUP:/$testCmd->{'current_group_user'}/g;
$arg =~ s/:UNAME_OTHER:/$testCmd->{'current_other_user'}/g;
$arg =~ s/:UGROUP:/$testCmd->{'current_group'}/g;
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Fri Aug 30 06:59:08 2013
@@ -234,5 +234,49 @@
in the cluster are taken over by Templeton launcher tasks.
</description>
</property>
+ <!--
+ <property>
+ <name>webhcat.proxyuser.#USER#.hosts</name>
+ <value>www.example.com,host2</value>
+ <description>
+ List of client hosts from which the '#USER#' user is allowed to perform
+ 'doAs' operations.
+ The '#USER#' must be replaced with the username of the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard, which means every host is allowed,
+ or a comma-separated list of hostnames.
+
+ If value is a blank string or webhcat.proxyuser.#USER#.hosts is missing,
+ no hosts will be allowed.
+
+ For multiple users copy this property and replace the user name
+ in the property name.
+ </description>
+ </property>
+ <property>
+ <name>webhcat.proxyuser.#USER#.groups</name>
+ <value>group1, group2</value>
+ <description>
+ List of groups the '#USER#' user is allowed to impersonate users
+ from to perform 'doAs' operations.
+
+ The '#USER#' must be replaced with the username of the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard, which means any doAs value is
+ allowed, or a comma-separated list of groups.
+
+ If value is an empty list or webhcat.proxyuser.#USER#.groups is missing,
+ every doAs call value will fail.
+
+ For multiple users copy this property and replace the user name
+ in the property name.
+
+ The username->usergroup mapping is performed using Hadoop API which is
+ controlled by hadoop.security.group.mapping property.
+ </description>
+ </property>
+-->
</configuration>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java Fri Aug 30 06:59:08 2013
@@ -138,6 +138,7 @@ public class AppConfig extends Configura
String hadoopConfDir = getHadoopConfDir();
for (String fname : HADOOP_CONF_FILENAMES)
loadOneFileConfig(hadoopConfDir, fname);
+ ProxyUserSupport.processProxyuserConfig(this);
}
public void startCleanup() {
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java Fri Aug 30 06:59:08 2013
@@ -41,8 +41,7 @@ import org.apache.hcatalog.templeton.too
* launch child jobs.
*/
public class LauncherDelegator extends TempletonDelegator {
- private static final Log LOG = LogFactory.getLog(Server.class);
- public static final String JAR_CLASS = TempletonControllerJob.class.getName();
+ private static final Log LOG = LogFactory.getLog(LauncherDelegator.class);
protected String runAs = null;
public LauncherDelegator(AppConfig appConf) {
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java Fri Aug 30 06:59:08 2013
@@ -19,6 +19,9 @@
package org.apache.hcatalog.templeton;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -26,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.DELETE;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
@@ -54,6 +58,7 @@ import org.apache.hcatalog.templeton.too
@Path("/v1")
public class Server {
public static final String VERSION = "v1";
+ public static final String DO_AS_PARAM = "doAs";
/**
* The status message. Always "ok"
@@ -113,6 +118,8 @@ public class Server {
private
@Context
UriInfo theUriInfo;
+ private @QueryParam(DO_AS_PARAM) String doAs;
+ private @Context HttpServletRequest request;
private static final Log LOG = LogFactory.getLog(Server.class);
@@ -161,7 +168,7 @@ public class Server {
verifyParam(exec, "exec");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.run(getUser(), exec, false, group, permissions);
+ return d.run(getDoAsUser(), exec, false, group, permissions);
}
/**
@@ -180,7 +187,7 @@ public class Server {
HcatDelegator d = new HcatDelegator(appConf, execService);
if (!TempletonUtils.isset(tablePattern))
tablePattern = "*";
- return d.listTables(getUser(), db, tablePattern);
+ return d.listTables(getDoAsUser(), db, tablePattern);
}
/**
@@ -200,7 +207,7 @@ public class Server {
desc.table = table;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.createTable(getUser(), db, desc);
+ return d.createTable(getDoAsUser(), db, desc);
}
/**
@@ -223,7 +230,7 @@ public class Server {
desc.newTable = newTable;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.createTableLike(getUser(), db, desc);
+ return d.createTableLike(getDoAsUser(), db, desc);
}
/**
@@ -245,9 +252,9 @@ public class Server {
HcatDelegator d = new HcatDelegator(appConf, execService);
if ("extended".equals(format))
- return d.descExtendedTable(getUser(), db, table);
+ return d.descExtendedTable(getDoAsUser(), db, table);
else
- return d.descTable(getUser(), db, table, false);
+ return d.descTable(getDoAsUser(), db, table, false);
}
/**
@@ -268,7 +275,7 @@ public class Server {
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.dropTable(getUser(), db, table, ifExists, group, permissions);
+ return d.dropTable(getDoAsUser(), db, table, ifExists, group, permissions);
}
/**
@@ -290,7 +297,7 @@ public class Server {
verifyDdlParam(newTable, "rename");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.renameTable(getUser(), db, oldTable, newTable, group, permissions);
+ return d.renameTable(getDoAsUser(), db, oldTable, newTable, group, permissions);
}
/**
@@ -310,7 +317,7 @@ public class Server {
verifyDdlParam(property, ":property");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descTableProperty(getUser(), db, table, property);
+ return d.descTableProperty(getDoAsUser(), db, table, property);
}
/**
@@ -328,7 +335,7 @@ public class Server {
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.listTableProperties(getUser(), db, table);
+ return d.listTableProperties(getDoAsUser(), db, table);
}
/**
@@ -350,7 +357,7 @@ public class Server {
desc.name = property;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.addOneTableProperty(getUser(), db, table, desc);
+ return d.addOneTableProperty(getDoAsUser(), db, table, desc);
}
/**
@@ -368,7 +375,7 @@ public class Server {
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.listPartitions(getUser(), db, table);
+ return d.listPartitions(getDoAsUser(), db, table);
}
/**
@@ -388,7 +395,7 @@ public class Server {
verifyParam(partition, ":partition");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descOnePartition(getUser(), db, table, partition);
+ return d.descOnePartition(getDoAsUser(), db, table, partition);
}
/**
@@ -409,7 +416,7 @@ public class Server {
verifyParam(partition, ":partition");
desc.partition = partition;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.addOnePartition(getUser(), db, table, desc);
+ return d.addOnePartition(getDoAsUser(), db, table, desc);
}
/**
@@ -431,8 +438,8 @@ public class Server {
verifyDdlParam(table, ":table");
verifyParam(partition, ":partition");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.dropPartition(getUser(), db, table, partition, ifExists,
- group, permissions);
+ return d.dropPartition(getDoAsUser(), db, table, partition, ifExists,
+ group, permissions);
}
/**
@@ -449,7 +456,7 @@ public class Server {
HcatDelegator d = new HcatDelegator(appConf, execService);
if (!TempletonUtils.isset(dbPattern))
dbPattern = "*";
- return d.listDatabases(getUser(), dbPattern);
+ return d.listDatabases(getDoAsUser(), dbPattern);
}
/**
@@ -465,7 +472,7 @@ public class Server {
verifyUser();
verifyDdlParam(db, ":db");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descDatabase(getUser(), db, "extended".equals(format));
+ return d.descDatabase(getDoAsUser(), db, "extended".equals(format));
}
/**
@@ -482,7 +489,7 @@ public class Server {
verifyDdlParam(db, ":db");
desc.database = db;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.createDatabase(getUser(), desc);
+ return d.createDatabase(getDoAsUser(), desc);
}
/**
@@ -503,8 +510,8 @@ public class Server {
if (TempletonUtils.isset(option))
verifyDdlParam(option, "option");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.dropDatabase(getUser(), db, ifExists, option,
- group, permissions);
+ return d.dropDatabase(getDoAsUser(), db, ifExists, option,
+ group, permissions);
}
/**
@@ -523,7 +530,7 @@ public class Server {
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.listColumns(getUser(), db, table);
+ return d.listColumns(getDoAsUser(), db, table);
}
/**
@@ -543,7 +550,7 @@ public class Server {
verifyParam(column, ":column");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descOneColumn(getUser(), db, table, column);
+ return d.descOneColumn(getDoAsUser(), db, table, column);
}
/**
@@ -566,7 +573,7 @@ public class Server {
desc.name = column;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.addOneColumn(getUser(), db, table, desc);
+ return d.addOneColumn(getDoAsUser(), db, table, desc);
}
/**
@@ -593,7 +600,7 @@ public class Server {
verifyParam(reducer, "reducer");
StreamingDelegator d = new StreamingDelegator(appConf);
- return d.run(getUser(), inputs, output, mapper, reducer,
+ return d.run(getDoAsUser(), inputs, output, mapper, reducer,
files, defines, cmdenvs, args,
statusdir, callback, getCompletedUrl());
}
@@ -619,7 +626,7 @@ public class Server {
verifyParam(mainClass, "class");
JarDelegator d = new JarDelegator(appConf);
- return d.run(getUser(),
+ return d.run(getDoAsUser(),
jar, mainClass,
libjars, files, args, defines,
statusdir, callback, getCompletedUrl());
@@ -644,7 +651,7 @@ public class Server {
throw new BadParam("Either execute or file parameter required");
PigDelegator d = new PigDelegator(appConf);
- return d.run(getUser(),
+ return d.run(getDoAsUser(),
execute, srcFile,
pigArgs, otherFiles,
statusdir, callback, getCompletedUrl());
@@ -668,7 +675,7 @@ public class Server {
throw new BadParam("Either execute or file parameter required");
HiveDelegator d = new HiveDelegator(appConf);
- return d.run(getUser(), execute, srcFile, defines,
+ return d.run(getDoAsUser(), execute, srcFile, defines,
statusdir, callback, getCompletedUrl());
}
@@ -685,7 +692,7 @@ public class Server {
verifyParam(jobid, ":jobid");
StatusDelegator d = new StatusDelegator(appConf);
- return d.run(getUser(), jobid);
+ return d.run(getDoAsUser(), jobid);
}
/**
@@ -701,7 +708,7 @@ public class Server {
verifyParam(jobid, ":jobid");
DeleteDelegator d = new DeleteDelegator(appConf);
- return d.run(getUser(), jobid);
+ return d.run(getDoAsUser(), jobid);
}
/**
@@ -716,7 +723,7 @@ public class Server {
verifyUser();
ListDelegator d = new ListDelegator(appConf);
- return d.run(getUser());
+ return d.run(getDoAsUser());
}
/**
@@ -734,16 +741,30 @@ public class Server {
/**
* Verify that we have a valid user. Throw an exception if invalid.
*/
- public void verifyUser()
- throws NotAuthorizedException {
- if (getUser() == null) {
+ public void verifyUser() throws NotAuthorizedException {
+ String requestingUser = getRequestingUser();
+ if (requestingUser == null) {
String msg = "No user found.";
if (!UserGroupInformation.isSecurityEnabled())
msg += " Missing " + PseudoAuthenticator.USER_NAME + " parameter.";
throw new NotAuthorizedException(msg);
}
+ if(doAs != null && !doAs.equals(requestingUser)) {
+ /*if doAs user is different than logged in user, need to check that
+ that logged in user is authorized to run as 'doAs'*/
+ ProxyUserSupport.validate(requestingUser, getRequestingHost(requestingUser, request), doAs);
+ }
+ }
+ /**
+ * All 'tasks' spawned by WebHCat should be run as this user. W/o doAs query parameter
+ * this is just the user making the request (or
+ * {@link org.apache.hadoop.security.authentication.client.PseudoAuthenticator#USER_NAME}
+ * query param).
+ * @return value of doAs query parameter or {@link #getRequestingUser()}
+ */
+ private String getDoAsUser() {
+ return doAs != null && !doAs.equals(getRequestingUser()) ? doAs : getRequestingUser();
}
-
/**
* Verify that the parameter exists. Throw an exception if invalid.
*/
@@ -777,16 +798,20 @@ public class Server {
if (!m.matches())
throw new BadParam("Invalid DDL identifier " + name);
}
-
/**
- * Get the user name from the security context.
+ * Get the user name from the security context, i.e. the user making the HTTP request.
+ * With simple/pseudo security mode this should return the
+ * value of user.name query param, in kerberos mode it's the kinit'ed user.
*/
- public String getUser() {
+ private String getRequestingUser() {
if (theSecurityContext == null)
return null;
if (theSecurityContext.getUserPrincipal() == null)
return null;
- return theSecurityContext.getUserPrincipal().getName();
+ //map hue/foo.bar@something.com->hue since user group checks
+ // and config files are in terms of short name
+ return UserGroupInformation.createRemoteUser(
+ theSecurityContext.getUserPrincipal().getName()).getShortUserName();
}
/**
@@ -800,4 +825,32 @@ public class Server {
return theUriInfo.getBaseUri() + VERSION
+ "/internal/complete/$jobId";
}
+ /**
+ * Returns canonical host name from which the request is made; used for doAs validation
+ */
+ private static String getRequestingHost(String requestingUser, HttpServletRequest request) {
+ final String unkHost = "???";
+ if(request == null) {
+ LOG.warn("request is null; cannot determine hostname");
+ return unkHost;
+ }
+ try {
+ String address = request.getRemoteAddr();//returns IP addr
+ if(address == null) {
+ LOG.warn(MessageFormat.format("Request remote address is NULL for user [{0}]", requestingUser));
+ return unkHost;
+ }
+
+ //Inet4Address/Inet6Address
+ String hostName = InetAddress.getByName(address).getCanonicalHostName();
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(MessageFormat.format("Resolved remote hostname: [{0}]", hostName));
+ }
+ return hostName;
+
+ } catch (UnknownHostException ex) {
+ LOG.warn(MessageFormat.format("Request remote address could not be resolved, {0}", ex.toString(), ex));
+ return unkHost;
+ }
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java Fri Aug 30 06:59:08 2013
@@ -27,7 +27,7 @@ public class UgiFactory {
private static ConcurrentHashMap<String, UserGroupInformation> userUgiMap =
new ConcurrentHashMap<String, UserGroupInformation>();
- static UserGroupInformation getUgi(String user) throws IOException {
+ public static UserGroupInformation getUgi(String user) throws IOException {
UserGroupInformation ugi = userUgiMap.get(user);
if (ugi == null) {
//create new ugi and add to map
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java Fri Aug 30 06:59:08 2013
@@ -19,6 +19,7 @@
package org.apache.hcatalog.templeton.tool;
import java.io.BufferedReader;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -68,31 +69,29 @@ public class HDFSStorage implements Temp
return;
}
PrintWriter out = null;
+ //todo: FileSystem#setPermission() - should this make sure to set 777 on jobs/ ?
+ Path keyfile= new Path(getPath(type) + "/" + id + "/" + key);
try {
- Path keyfile = new Path(getPath(type) + "/" + id + "/" + key);
// This will replace the old value if there is one
// Overwrite the existing file
out = new PrintWriter(new OutputStreamWriter(fs.create(keyfile)));
out.write(val);
- } catch (IOException e) {
- LOG.info("Couldn't write to " + getPath(type) + "/" + id + ": "
- + e.getMessage());
+ out.flush();
+ } catch (Exception e) {
+ String errMsg = "Couldn't write to " + keyfile + ": " + e.getMessage();
+ LOG.error(errMsg, e);
+ throw new NotFoundException(errMsg, e);
} finally {
- try {
- out.flush();
- out.close();
- } catch (Exception e) {
- // fail
- }
+ close(out);
}
}
@Override
public String getField(Type type, String id, String key) {
BufferedReader in = null;
+ Path p = new Path(getPath(type) + "/" + id + "/" + key);
try {
- in = new BufferedReader(new InputStreamReader(fs.open(new Path(getPath(type) + "/" +
- id + "/" + key))));
+ in = new BufferedReader(new InputStreamReader(fs.open(p)));
String line = null;
String val = "";
while ((line = in.readLine()) != null) {
@@ -102,15 +101,10 @@ public class HDFSStorage implements Temp
val += line;
}
return val;
- } catch (IOException e) {
- LOG.trace("Couldn't find " + getPath(type) + "/" + id + "/" + key
- + ": " + e.getMessage());
+ } catch (Exception e) {
+ LOG.info("Couldn't find " + p + ": " + e.getMessage(), e);
} finally {
- try {
- in.close();
- } catch (Exception e) {
- // fail
- }
+ close(in);
}
return null;
}
@@ -119,8 +113,9 @@ public class HDFSStorage implements Temp
public Map<String, String> getFields(Type type, String id) {
HashMap<String, String> map = new HashMap<String, String>();
BufferedReader in = null;
+ Path p = new Path(getPath(type) + "/" + id);
try {
- for (FileStatus status : fs.listStatus(new Path(getPath(type) + "/" + id))) {
+ for (FileStatus status : fs.listStatus(p)) {
in = new BufferedReader(new InputStreamReader(fs.open(status.getPath())));
String line = null;
String val = "";
@@ -133,23 +128,20 @@ public class HDFSStorage implements Temp
map.put(status.getPath().getName(), val);
}
} catch (IOException e) {
- LOG.trace("Couldn't find " + getPath(type) + "/" + id);
+ LOG.trace("Couldn't find " + p);
} finally {
- try {
- in.close();
- } catch (Exception e) {
- // fail
- }
+ close(in);
}
return map;
}
@Override
public boolean delete(Type type, String id) throws NotFoundException {
+ Path p = new Path(getPath(type) + "/" + id);
try {
- fs.delete(new Path(getPath(type) + "/" + id), true);
+ fs.delete(p, true);
} catch (IOException e) {
- throw new NotFoundException("Node " + id + " was not found: " +
+ throw new NotFoundException("Node " + p + " was not found: " +
e.getMessage());
}
return false;
@@ -251,4 +243,15 @@ public class HDFSStorage implements Temp
}
return typepath;
}
+ private void close(Closeable is) {
+ if(is == null) {
+ return;
+ }
+ try {
+ is.close();
+ }
+ catch (IOException ex) {
+ LOG.trace("Failed to close InputStream: " + ex.getMessage());
+ }
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java Fri Aug 30 06:59:08 2013
@@ -27,4 +27,7 @@ public class NotFoundException extends E
public NotFoundException(String msg) {
super(msg);
}
+ public NotFoundException(String msg, Throwable rootCause) {
+ super(msg, rootCause);
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java?rev=1518894&r1=1518893&r2=1518894&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java Fri Aug 30 06:59:08 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hcatalog.templeton.UgiFactory;
/**
* General utility methods.
@@ -210,23 +211,19 @@ public class TempletonUtils {
}
}
- public static Path hadoopFsPath(String fname, Configuration conf, String user)
- throws URISyntaxException, FileNotFoundException, IOException,
+ public static Path hadoopFsPath(final String fname, final Configuration conf, String user)
+ throws URISyntaxException, IOException,
InterruptedException {
if (fname == null || conf == null) {
return null;
}
- final Configuration fConf = new Configuration(conf);
- final String finalFName = new String(fname);
-
- UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ UserGroupInformation ugi = UgiFactory.getUgi(user);
final FileSystem defaultFs =
ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run()
- throws URISyntaxException, FileNotFoundException, IOException,
- InterruptedException {
- return FileSystem.get(new URI(finalFName), fConf);
+ throws URISyntaxException, IOException, InterruptedException {
+ return FileSystem.get(new URI(fname), conf);
}
});