You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/10/01 02:54:17 UTC
svn commit: r1527840 - in /hive/trunk/hcatalog/webhcat/svr/src: main/config/
main/java/org/apache/hive/hcatalog/templeton/
main/java/org/apache/hive/hcatalog/templeton/tool/
test/java/org/apache/hive/hcatalog/templeton/
test/java/org/apache/hive/hcatal...
Author: thejas
Date: Tue Oct 1 00:54:17 2013
New Revision: 1527840
URL: http://svn.apache.org/r1527840
Log:
HIVE-5066: [WebHCat] Other code fixes for Windows (Daniel Dai via Thejas Nair)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java
hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Tue Oct 1 00:54:17 2013
@@ -80,32 +80,38 @@
</property>
<property>
+ <name>templeton.python</name>
+ <value>${env.PYTHON_CMD}</value>
+ <description>The path to the python executable.</description>
+ </property>
+
+ <property>
<name>templeton.pig.archive</name>
- <value>hdfs:///apps/templeton/pig-0.10.1.tar.gz</value>
+ <value></value>
<description>The path to the Pig archive.</description>
</property>
<property>
<name>templeton.pig.path</name>
- <value>pig-0.10.1.tar.gz/pig-0.10.1/bin/pig</value>
+ <value>pig-0.11.1.tar.gz/pig-0.11.1/bin/pig</value>
<description>The path to the Pig executable.</description>
</property>
<property>
<name>templeton.hcat</name>
- <value>${env.HCAT_PREFIX}/bin/hcat</value>
+ <value>${env.HCAT_PREFIX}/bin/hcat.py</value>
<description>The path to the hcatalog executable.</description>
</property>
<property>
<name>templeton.hive.archive</name>
- <value>hdfs:///apps/templeton/hive-0.10.0.tar.gz</value>
+ <value></value>
<description>The path to the Hive archive.</description>
</property>
<property>
<name>templeton.hive.path</name>
- <value>hive-0.10.0.tar.gz/hive-0.10.0/bin/hive</value>
+ <value>hive-0.11.0.tar.gz/hive-0.11.0/bin/hive</value>
<description>The path to the Hive executable.</description>
</property>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Tue Oct 1 00:54:17 2013
@@ -87,6 +87,7 @@ public class AppConfig extends Configura
public static final String HADOOP_NAME = "templeton.hadoop";
public static final String HADOOP_CONF_DIR = "templeton.hadoop.conf.dir";
public static final String HCAT_NAME = "templeton.hcat";
+ public static final String PYTHON_NAME = "templeton.python";
public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
public static final String HIVE_PATH_NAME = "templeton.hive.path";
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
@@ -181,6 +182,7 @@ public class AppConfig extends Configura
public String hadoopQueueName() { return get(HADOOP_QUEUE_NAME); }
public String clusterHadoop() { return get(HADOOP_NAME); }
public String clusterHcat() { return get(HCAT_NAME); }
+ public String clusterPython() { return get(PYTHON_NAME); }
public String pigPath() { return get(PIG_PATH_NAME); }
public String pigArchive() { return get(PIG_ARCHIVE_NAME); }
public String hivePath() { return get(HIVE_PATH_NAME); }
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java Tue Oct 1 00:54:17 2013
@@ -18,12 +18,18 @@
*/
package org.apache.hive.hcatalog.templeton;
+import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.Semaphore;
import org.apache.commons.exec.CommandLine;
@@ -33,6 +39,38 @@ import org.apache.commons.exec.ExecuteWa
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
+
+class StreamOutputWriter extends Thread
+{
+ InputStream is;
+ String type;
+ PrintWriter out;
+
+ StreamOutputWriter(InputStream is, String type, OutputStream outStream)
+ {
+ this.is = is;
+ this.type = type;
+ this.out = new PrintWriter(outStream, true);
+ }
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ BufferedReader br =
+ new BufferedReader(new InputStreamReader(is));
+ String line = null;
+ while ( (line = br.readLine()) != null){
+ out.println(line);
+ }
+ } catch (IOException ioe)
+ {
+ ioe.printStackTrace();
+ }
+ }
+}
/**
* Execute a local program. This is a singleton service that will
@@ -45,6 +83,9 @@ public class ExecServiceImpl implements
private static volatile ExecServiceImpl theSingleton;
+ /** Windows CreateProcess synchronization object */
+ private static final Object WindowsProcessLaunchLock = new Object();
+
/**
* Retrieve the singleton.
*/
@@ -133,7 +174,54 @@ public class ExecServiceImpl implements
LOG.info("Running: " + cmd);
ExecBean res = new ExecBean();
- res.exitcode = executor.execute(cmd, execEnv(env));
+
+ if(Shell.WINDOWS){
+ //The default executor is sometimes causing failure on windows. hcat
+ // command sometimes returns non zero exit status with it. It seems
+ // to hit some race conditions on windows.
+ env = execEnv(env);
+ String[] envVals = new String[env.size()];
+ int i=0;
+ for( Entry<String, String> kv : env.entrySet()){
+ envVals[i++] = kv.getKey() + "=" + kv.getValue();
+ LOG.info("Setting " + kv.getKey() + "=" + kv.getValue());
+ }
+
+ Process proc;
+ synchronized (WindowsProcessLaunchLock) {
+ // To workaround the race condition issue with child processes
+ // inheriting unintended handles during process launch that can
+ // lead to hangs on reading output and error streams, we
+ // serialize process creation. More info available at:
+ // http://support.microsoft.com/kb/315939
+ proc = Runtime.getRuntime().exec(cmd.toStrings(), envVals);
+ }
+
+ //consume stderr
+ StreamOutputWriter errorGobbler = new
+ StreamOutputWriter(proc.getErrorStream(), "ERROR", errStream);
+
+ //consume stdout
+ StreamOutputWriter outputGobbler = new
+ StreamOutputWriter(proc.getInputStream(), "OUTPUT", outStream);
+
+ //start collecting input streams
+ errorGobbler.start();
+ outputGobbler.start();
+ //execute
+ try{
+ res.exitcode = proc.waitFor();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ //flush
+ errorGobbler.out.flush();
+ outputGobbler.out.flush();
+ }
+ else {
+ res.exitcode = executor.execute(cmd, execEnv(env));
+ }
+
String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME);
res.stdout = outStream.toString(enc);
res.stderr = errStream.toString(enc);
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java Tue Oct 1 00:54:17 2013
@@ -67,7 +67,11 @@ public class HcatDelegator extends Launc
Map<String, String> env = TempletonUtils.hadoopUserEnv(user, cp);
proxy.addEnv(env);
proxy.addArgs(args);
- return execService.run(appConf.clusterHcat(), args, env);
+ if (appConf.clusterHcat().toLowerCase().endsWith(".py")) {
+ return execService.run(appConf.clusterPython(), args, env);
+ } else {
+ return execService.run(appConf.clusterHcat(), args, env);
+ }
} catch (InterruptedException e) {
throw new IOException(e);
} finally {
@@ -79,8 +83,12 @@ public class HcatDelegator extends Launc
private List<String> makeArgs(String exec, boolean format,
String group, String permissions) {
ArrayList<String> args = new ArrayList<String>();
+ if (appConf.clusterHcat().toLowerCase().endsWith(".py")) {
+ // hcat.py will become the first argument pass to command "python"
+ args.add(appConf.clusterHcat());
+ }
args.add("-e");
- args.add(exec);
+ args.add('"' + exec + '"');
if (TempletonUtils.isset(group)) {
args.add("-g");
args.add(group);
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Oct 1 00:54:17 2013
@@ -64,6 +64,7 @@ public class HiveDelegator extends Launc
try {
args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog));
args.add("--");
+ TempletonUtils.addCmdForWindows(args);
args.add(appConf.hivePath());
args.add("--service");
@@ -122,8 +123,11 @@ public class HiveDelegator extends Launc
args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
enablelog, JobType.HIVE));
- args.add("-archives");
- args.add(appConf.hiveArchive());
+ if (appConf.hiveArchive() != null && !appConf.hiveArchive().equals(""))
+ {
+ args.add("-archives");
+ args.add(appConf.hiveArchive());
+ }
return args;
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Oct 1 00:54:17 2013
@@ -68,6 +68,7 @@ public class JarDelegator extends Launch
args.addAll(makeLauncherArgs(appConf, statusdir,
completedUrl, allFiles, enablelog, jobType));
args.add("--");
+ TempletonUtils.addCmdForWindows(args);
args.add(appConf.clusterHadoop());
args.add("jar");
args.add(TempletonUtils.hadoopFsPath(jar, appConf, runAs).getName());
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Oct 1 00:54:17 2013
@@ -70,10 +70,14 @@ public class PigDelegator extends Launch
}
args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG));
- args.add("-archives");
- args.add(appConf.pigArchive());
+ if (appConf.pigArchive() != null && !appConf.pigArchive().equals(""))
+ {
+ args.add("-archives");
+ args.add(appConf.pigArchive());
+ }
args.add("--");
+ TempletonUtils.addCmdForWindows(args);
args.add(appConf.pigPath());
//the token file location should be first argument of pig
args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java Tue Oct 1 00:54:17 2013
@@ -98,7 +98,7 @@ public class HDFSCleanup extends Thread
// cycle fails, it'll try again on the next cycle.
try {
if (fs == null) {
- fs = FileSystem.get(appConf);
+ fs = new Path(storage_root).getFileSystem(appConf);
}
checkFiles(fs);
} catch (Exception e) {
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java Tue Oct 1 00:54:17 2013
@@ -210,7 +210,7 @@ public class HDFSStorage implements Temp
public void openStorage(Configuration config) throws IOException {
storage_root = config.get(TempletonStorage.STORAGE_ROOT);
if (fs == null) {
- fs = FileSystem.get(config);
+ fs = new Path(storage_root).getFileSystem(config);
}
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Tue Oct 1 00:54:17 2013
@@ -48,12 +48,15 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.templeton.BadParam;
import org.apache.hive.hcatalog.templeton.LauncherDelegator;
/**
@@ -104,6 +107,9 @@ public class TempletonControllerJob exte
ArrayList<String> removeEnv = new ArrayList<String>();
removeEnv.add("HADOOP_ROOT_LOGGER");
+ removeEnv.add("hadoop-command");
+ removeEnv.add("CLASS");
+ removeEnv.add("mapredcommand");
Map<String, String> env = TempletonUtils.hadoopUserEnv(user,
overrideClasspath);
List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
@@ -112,7 +118,15 @@ public class TempletonControllerJob exte
if (tokenFile != null) {
//Token is available, so replace the placeholder
+ tokenFile = tokenFile.replaceAll("\"", "");
String tokenArg = "mapreduce.job.credentials.binary=" + tokenFile;
+ if (Shell.WINDOWS) {
+ try {
+ tokenArg = TempletonUtils.quoteForWindows(tokenArg);
+ } catch (BadParam e) {
+ throw new IOException("cannot pass " + tokenFile + " to mapreduce.job.credentials.binary", e);
+ }
+ }
for(int i=0; i<jarArgsList.size(); i++){
String newArg =
jarArgsList.get(i).replace(TOKEN_FILE_ARG_PLACEHOLDER, tokenArg);
@@ -211,9 +225,9 @@ public class TempletonControllerJob exte
pool.execute(w);
}
- private KeepAlive startCounterKeepAlive(ExecutorService pool, Context cnt)
+ private KeepAlive startCounterKeepAlive(ExecutorService pool, Context context)
throws IOException {
- KeepAlive k = new KeepAlive(cnt);
+ KeepAlive k = new KeepAlive(context);
pool.execute(k);
return k;
}
@@ -297,20 +311,25 @@ public class TempletonControllerJob exte
}
}
- private static class KeepAlive implements Runnable {
- private final Mapper.Context cnt;
- private volatile boolean sendReport;
+ public static class KeepAlive implements Runnable {
+ private Context context;
+ public boolean sendReport;
- public KeepAlive(Mapper.Context cnt) {
- this.cnt = cnt;
+ public KeepAlive(Context context)
+ {
this.sendReport = true;
+ this.context = context;
}
@Override
public void run() {
try {
while (sendReport) {
- cnt.progress();
+ // Periodically report progress on the Context object
+ // to prevent TaskTracker from killing the Templeton
+ // Controller task
+ context.progress();
+ System.err.println("KeepAlive Heart beat");
Thread.sleep(KEEP_ALIVE_MSEC);
}
} catch (InterruptedException e) {
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java Tue Oct 1 00:54:17 2013
@@ -22,12 +22,16 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* Execute a local program. This is a singleton service that will
* execute a programs on the local box.
*/
public class TrivialExecService {
private static volatile TrivialExecService theSingleton;
+ private static final Log LOG = LogFactory.getLog(TrivialExecService.class);
/**
* Retrieve the singleton.
@@ -41,11 +45,7 @@ public class TrivialExecService {
public Process run(List<String> cmd, List<String> removeEnv,
Map<String, String> environmentVariables)
throws IOException {
- System.err.println("templeton: starting " + cmd);
- System.err.print("With environment variables: ");
- for (Map.Entry<String, String> keyVal : environmentVariables.entrySet()) {
- System.err.println(keyVal.getKey() + "=" + keyVal.getValue());
- }
+ logDebugCmd(cmd, environmentVariables);
ProcessBuilder pb = new ProcessBuilder(cmd);
for (String key : removeEnv)
pb.environment().remove(key);
@@ -53,4 +53,20 @@ public class TrivialExecService {
return pb.start();
}
+ private void logDebugCmd(List<String> cmd,
+ Map<String, String> environmentVariables) {
+ if(!LOG.isDebugEnabled()){
+ return;
+ }
+ LOG.debug("starting " + cmd);
+ LOG.debug("With environment variables: " );
+ for(Map.Entry<String, String> keyVal : environmentVariables.entrySet()){
+ LOG.debug(keyVal.getKey() + "=" + keyVal.getValue());
+ }
+ LOG.debug("With environment variables already set: " );
+ Map<String, String> env = System.getenv();
+ for (String envName : env.keySet()) {
+ LOG.debug(envName + "=" + env.get(envName));
+ }
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java Tue Oct 1 00:54:17 2013
@@ -31,7 +31,7 @@ public class TestServer extends TestCase
MockServer server;
public void setUp() {
- new Main(null); // Initialize the config
+ new Main(new String[]{}); // Initialize the config
server = new MockServer();
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Tue Oct 1 00:54:17 2013
@@ -159,6 +159,12 @@ public class TestTempletonUtils {
@Test
public void testHadoopFsFilename() {
try {
+ String tmpFileName1 = "/tmp/testHadoopFsListAsArray1";
+ String tmpFileName2 = "/tmp/testHadoopFsListAsArray2";
+ File tmpFile1 = new File(tmpFileName1);
+ File tmpFile2 = new File(tmpFileName2);
+ tmpFile1.createNewFile();
+ tmpFile2.createNewFile();
Assert.assertEquals(null, TempletonUtils.hadoopFsFilename(null, null, null));
Assert.assertEquals(null,
TempletonUtils.hadoopFsFilename(tmpFile.toURI().toString(), null, null));
@@ -188,14 +194,22 @@ public class TestTempletonUtils {
@Test
public void testHadoopFsListAsArray() {
try {
+ String tmpFileName1 = "/tmp/testHadoopFsListAsArray1";
+ String tmpFileName2 = "/tmp/testHadoopFsListAsArray2";
+ File tmpFile1 = new File(tmpFileName1);
+ File tmpFile2 = new File(tmpFileName2);
+ tmpFile1.createNewFile();
+ tmpFile2.createNewFile();
Assert.assertTrue(TempletonUtils.hadoopFsListAsArray(null, null, null) == null);
- Assert.assertTrue(TempletonUtils.hadoopFsListAsArray(
- tmpFile.toURI().toString() + "," + usrFile.toString(), null, null) == null);
- String[] tmp2 = TempletonUtils.hadoopFsListAsArray(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
- new Configuration(), null);
- Assert.assertEquals(tmpFile.toURI().toString(), tmp2[0]);
- Assert.assertEquals(usrFile.toURI().toString(), tmp2[1]);
+ Assert.assertTrue(TempletonUtils.hadoopFsListAsArray(tmpFileName1 + "," + tmpFileName2,
+ null, null) == null);
+ String[] tmp2
+ = TempletonUtils.hadoopFsListAsArray(tmpFileName1 + "," + tmpFileName2,
+ new Configuration(), null);
+ Assert.assertEquals("file:" + tmpFileName1, tmp2[0]);
+ Assert.assertEquals("file:" + tmpFileName2, tmp2[1]);
+ tmpFile1.delete();
+ tmpFile2.delete();
} catch (FileNotFoundException e) {
Assert.fail("Couldn't find name for " + tmpFile.toURI().toString());
} catch (Exception e) {
@@ -218,15 +232,18 @@ public class TestTempletonUtils {
@Test
public void testHadoopFsListAsString() {
try {
+ String tmpFileName1 = "/tmp/testHadoopFsListAsString1";
+ String tmpFileName2 = "/tmp/testHadoopFsListAsString2";
+ File tmpFile1 = new File(tmpFileName1);
+ File tmpFile2 = new File(tmpFileName2);
+ tmpFile1.createNewFile();
+ tmpFile2.createNewFile();
Assert.assertTrue(TempletonUtils.hadoopFsListAsString(null, null, null) == null);
- Assert.assertTrue(TempletonUtils.hadoopFsListAsString(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
+ Assert.assertTrue(TempletonUtils.hadoopFsListAsString("/tmp,/usr",
null, null) == null);
- Assert.assertEquals(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
- TempletonUtils.hadoopFsListAsString(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
- new Configuration(), null));
+ Assert.assertEquals("file:" + tmpFileName1 + ",file:" + tmpFileName2,
+ TempletonUtils.hadoopFsListAsString
+ (tmpFileName1 + "," + tmpFileName2, new Configuration(), null));
} catch (FileNotFoundException e) {
Assert.fail("Couldn't find name for " + tmpFile.toURI().toString());
} catch (Exception e) {