You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2014/05/12 20:59:50 UTC
svn commit: r1594054 - in /hive/trunk: hcatalog/webhcat/svr/src/main/config/
hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/
hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/
packaging/src/main/asse...
Author: thejas
Date: Mon May 12 18:59:49 2014
New Revision: 1594054
URL: http://svn.apache.org/r1594054
Log:
HIVE-6768 : remove hcatalog/webhcat/svr/src/main/config/override-container-log4j.properties (Eugene Koifman via Thejas Nair)
Removed:
hive/trunk/hcatalog/webhcat/svr/src/main/config/override-container-log4j.properties
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java
hive/trunk/packaging/src/main/assembly/bin.xml
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Mon May 12 18:59:49 2014
@@ -25,9 +25,6 @@ public interface JobSubmissionConstants
public static final String JOB_TYPE = "templeton.jobtype";
public static final String JAR_ARGS_NAME = "templeton.args";
public static final String OVERRIDE_CLASSPATH = "templeton.override-classpath";
- public static final String OVERRIDE_CONTAINER_LOG4J_PROPS = "override.containerLog4j";
- //name of file
- static final String CONTAINER_LOG4J_PROPS = "override-container-log4j.properties";
public static final String STDOUT_FNAME = "stdout";
public static final String STDERR_FNAME = "stderr";
public static final String EXIT_FNAME = "exit";
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Mon May 12 18:59:49 2014
@@ -103,9 +103,7 @@ public class LaunchMapper extends Mapper
List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
- boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
- false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
- return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
+ return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env);
}
/**
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Mon May 12 18:59:49 2014
@@ -74,87 +74,6 @@ import org.apache.thrift.TException;
@InterfaceAudience.Private
public class TempletonControllerJob extends Configured implements Tool, JobSubmissionConstants {
private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class);
- //file to add to DistributedCache
- private static URI overrideLog4jURI = null;
- private static boolean overrideContainerLog4jProps;
- //Jar cmd submission likely will be affected, Pig likely not
- private static final String affectedMsg = "Monitoring of Hadoop jobs submitted through WebHCat " +
- "may be affected.";
- private static final String TMP_DIR_PROP = "hadoop.tmp.dir";
-
- /**
- * Copy the file from local file system to tmp dir
- */
- private static URI copyLog4JtoFileSystem(final String localFile) throws IOException,
- InterruptedException {
- UserGroupInformation ugi = UserGroupInformation.getLoginUser();
- return ugi.doAs(new PrivilegedExceptionAction<URI>() {
- @Override
- public URI run() throws IOException {
- AppConfig appConfig = Main.getAppConfigInstance();
- String fsTmpDir = appConfig.get(TMP_DIR_PROP);
- if(fsTmpDir == null || fsTmpDir.length() <= 0) {
- LOG.warn("Could not find 'hadoop.tmp.dir'; " + affectedMsg);
- return null;
- }
- FileSystem fs = FileSystem.get(appConfig);
- Path dirPath = new Path(fsTmpDir);
- if(!fs.exists(dirPath)) {
- LOG.warn(dirPath + " does not exist; " + affectedMsg);
- return null;
- }
- Path dst = fs.makeQualified(new Path(fsTmpDir, CONTAINER_LOG4J_PROPS));
- fs.copyFromLocalFile(new Path(localFile), dst);
- //make readable by all users since TempletonControllerJob#run() is run as submitting user
- fs.setPermission(dst, new FsPermission((short)0644));
- return dst.toUri();
- }
- });
- }
- /**
- * local file system
- * @return
- */
- private static String getLog4JPropsLocal() {
- return AppConfig.getWebhcatConfDir() + File.separator + CONTAINER_LOG4J_PROPS;
- }
- static {
- //initialize once-per-JVM (i.e. one running WebHCat server) state and log it once since it's
- // the same for every job
- try {
- //safe (thread) publication
- // http://docs.oracle.com/javase/specs/jls/se5.0/html/execution.html#12.4.2
- LOG.info("Using Hadoop Version: " + ShimLoader.getMajorVersion());
- overrideContainerLog4jProps = "0.23".equals(ShimLoader.getMajorVersion());
- if(overrideContainerLog4jProps) {
- //see detailed note in CONTAINER_LOG4J_PROPS file
- LOG.info(AppConfig.WEBHCAT_CONF_DIR + "=" + AppConfig.getWebhcatConfDir());
- File localFile = new File(getLog4JPropsLocal());
- if(localFile.exists()) {
- LOG.info("Found " + localFile.getAbsolutePath() + " to use for job submission.");
- try {
- overrideLog4jURI = copyLog4JtoFileSystem(getLog4JPropsLocal());
- LOG.info("Job submission will use log4j.properties=" + overrideLog4jURI);
- }
- catch(IOException ex) {
- LOG.warn("Will not add " + CONTAINER_LOG4J_PROPS + " to Distributed Cache. " +
- "Some fields in job status may be unavailable", ex);
- }
- }
- else {
- LOG.warn("Could not find " + localFile.getAbsolutePath() + ". " + affectedMsg);
- }
- }
- }
- catch(Throwable t) {
- //this intentionally doesn't use TempletonControllerJob.class.getName() to be able to
- //log errors which may be due to class loading
- String msg = "org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob is not " +
- "properly initialized. " + affectedMsg;
- LOG.error(msg, t);
- }
- }
-
private final boolean secureMetastoreAccess;
/**
@@ -191,10 +110,6 @@ public class TempletonControllerJob exte
conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args));
String user = UserGroupInformation.getCurrentUser().getShortUserName();
conf.set("user.name", user);
- if(overrideContainerLog4jProps && overrideLog4jURI != null) {
- //must be done before Job object is created
- conf.set(OVERRIDE_CONTAINER_LOG4J_PROPS, Boolean.TRUE.toString());
- }
Job job = new Job(conf);
job.setJarByClass(LaunchMapper.class);
job.setJobName(TempletonControllerJob.class.getSimpleName());
@@ -202,20 +117,6 @@ public class TempletonControllerJob exte
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(SingleInputFormat.class);
- if(overrideContainerLog4jProps && overrideLog4jURI != null) {
- FileSystem fs = FileSystem.get(conf);
- if(fs.exists(new Path(overrideLog4jURI))) {
- ShimLoader.getHadoopShims().getWebHCatShim(conf, UgiFactory.getUgi(user)).addCacheFile(
- overrideLog4jURI, job);
- LOG.debug("added " + overrideLog4jURI + " to Dist Cache");
- }
- else {
- //in case this file was deleted by someone issue a warning but don't try to add to
- // DistributedCache as that will throw and fail job submission
- LOG.warn("Cannot find " + overrideLog4jURI + " which is created on WebHCat startup/job " +
- "submission. " + affectedMsg);
- }
- }
NullOutputFormat<NullWritable, NullWritable> of = new NullOutputFormat<NullWritable, NullWritable>();
job.setOutputFormatClass(of.getClass());
@@ -238,10 +139,6 @@ public class TempletonControllerJob exte
LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() +
" user=" + user);
}
- if(overrideContainerLog4jProps && overrideLog4jURI == null) {
- //do this here so that log msg has JobID
- LOG.warn("Could not override container log4j properties for " + submittedJobId);
- }
return 0;
}
private String addHMSToken(Job job, String user) throws IOException, InterruptedException,
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java Mon May 12 18:59:49 2014
@@ -39,10 +39,8 @@ import java.util.Map;
* not available on every node in the cluster (outside webhcat jar)
*/
final class TrivialExecService {
- //with default log4j config, this output ends up in 'syslog' of the LaunchMapper task
private static final Log LOG = LogFactory.getLog(TrivialExecService.class);
private static volatile TrivialExecService theSingleton;
- private static final String HADOOP_CLIENT_OPTS = "HADOOP_CLIENT_OPTS";
/**
* Retrieve the singleton.
*/
@@ -51,31 +49,9 @@ final class TrivialExecService {
theSingleton = new TrivialExecService();
return theSingleton;
}
- /**
- * See {@link JobSubmissionConstants#CONTAINER_LOG4J_PROPS} file for details.
- */
- private static void hadoop2LogRedirect(ProcessBuilder processBuilder) {
- Map<String, String> env = processBuilder.environment();
- if(!env.containsKey(HADOOP_CLIENT_OPTS)) {
- return;
- }
- String hcopts = env.get(HADOOP_CLIENT_OPTS);
- if(!hcopts.contains("log4j.configuration=container-log4j.properties")) {
- return;
- }
- //TempletonControllerJob ensures that this file is in DistributedCache
- File log4jProps = new File(JobSubmissionConstants.CONTAINER_LOG4J_PROPS);
- hcopts = hcopts.replace("log4j.configuration=container-log4j.properties",
- "log4j.configuration=file://" + new Path(log4jProps.getAbsolutePath()).toUri().toString());
- //helps figure out what log4j is doing, but may confuse
- //some jobs due to extra output to stdout
- //hcopts = hcopts + " -Dlog4j.debug=true";
- env.put(HADOOP_CLIENT_OPTS, hcopts);
- }
public Process run(List<String> cmd, List<String> removeEnv,
- Map<String, String> environmentVariables, boolean overrideContainerLog4jProps)
- throws IOException {
- LOG.info("run(cmd, removeEnv, environmentVariables, " + overrideContainerLog4jProps + ")");
+ Map<String, String> environmentVariables) throws IOException {
+ LOG.info("run(cmd, removeEnv, environmentVariables)");
LOG.info("Starting cmd: " + cmd);
ProcessBuilder pb = new ProcessBuilder(cmd);
for (String key : removeEnv) {
@@ -85,9 +61,6 @@ final class TrivialExecService {
pb.environment().remove(key);
}
pb.environment().putAll(environmentVariables);
- if(overrideContainerLog4jProps) {
- hadoop2LogRedirect(pb);
- }
logDebugInfo("Starting process with env:", pb.environment());
return pb.start();
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java Mon May 12 18:59:49 2014
@@ -38,7 +38,7 @@ public class TestTrivialExecService {
Process process = TrivialExecService.getInstance()
.run(list,
new ArrayList<String>(),
- new HashMap<String, String>(),false);
+ new HashMap<String, String>());
out = new BufferedReader(new InputStreamReader(
process.getInputStream()));
err = new BufferedReader(new InputStreamReader(
Modified: hive/trunk/packaging/src/main/assembly/bin.xml
URL: http://svn.apache.org/viewvc/hive/trunk/packaging/src/main/assembly/bin.xml?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/packaging/src/main/assembly/bin.xml (original)
+++ hive/trunk/packaging/src/main/assembly/bin.xml Mon May 12 18:59:49 2014
@@ -272,7 +272,6 @@
<includes>
<include>webhcat-default.xml</include>
<include>webhcat-log4j.properties</include>
- <include>override-container-log4j.properties</include>
</includes>
<outputDirectory>hcatalog/etc/webhcat</outputDirectory>
</fileSet>
Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java?rev=1594054&r1=1594053&r2=1594054&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java Mon May 12 18:59:49 2014
@@ -37,12 +37,8 @@ public class WebHCatJTShim23 implements
try {
jc = ugi.doAs(new PrivilegedExceptionAction<JobClient>() {
public JobClient run() throws IOException, InterruptedException {
- return ugi.doAs(new PrivilegedExceptionAction<JobClient>() {
- public JobClient run() throws IOException {
- //create this in doAs() so that it gets a security context based passed in 'ugi'
- return new JobClient(conf);
- }
- });
+ //create this in doAs() so that it gets a security context based passed in 'ugi'
+ return new JobClient(conf);
}
});
}