svn commit: r1640981 - in /hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton: AppConfig.java tool/TempletonUtils.java tool/TrivialExecService.java

[ek...@apache.org]

Author: ekoifman
Date: Fri Nov 21 19:24:54 2014
New Revision: 1640981

URL: http://svn.apache.org/r1640981
Log:
HIVE-8877 improve context logging during job submission via WebHCat (reviewed by Thejas Nair)

Modified:
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1640981&r1=1640980&r2=1640981&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Fri Nov 21 19:24:54 2014
@@ -28,12 +28,14 @@ import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.StringTokenizer;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.SystemVariables;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hive.hcatalog.templeton.tool.JobState;
@@ -224,14 +226,8 @@ public class AppConfig extends Configura
    * support/debugging.  Later it may be worth adding a REST call which will return this data.
    */
   private String dumpEnvironent() {
-    StringBuilder sb = new StringBuilder("WebHCat environment:\n");
-    Map<String, String> env = System.getenv();
-    List<String> propKeys = new ArrayList<String>(env.keySet());
-    Collections.sort(propKeys);
-    for(String propKey : propKeys) {
-      sb.append(propKey).append('=').append(env.get(propKey)).append('\n');
-    }
-    sb.append("Configration properties: \n");
+    StringBuilder sb = TempletonUtils.dumpPropMap("========WebHCat System.getenv()========", System.getenv());
+    sb.append("START========WebHCat AppConfig.iterator()========: \n");
     Iterator<Map.Entry<String, String>> configIter = this.iterator();
     List<Map.Entry<String, String>> configVals = new ArrayList<Map.Entry<String, String>>();
     while(configIter.hasNext()) {
@@ -245,8 +241,19 @@ public class AppConfig extends Configura
     });
     for(Map.Entry<String, String> entry : configVals) {
       //use get() to make sure variable substitution works
-      sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
+      if(entry.getKey().toLowerCase().contains("path")) {
+        StringTokenizer st = new StringTokenizer(get(entry.getKey()), File.pathSeparator);
+        sb.append(entry.getKey()).append("=\n");
+        while(st.hasMoreTokens()) {
+          sb.append("    ").append(st.nextToken()).append(File.pathSeparator).append('\n');
+        }
+      }
+      else {
+        sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
+      }
     }
+    sb.append("END========WebHCat AppConfig.iterator()========: \n");
+    sb.append(TempletonUtils.dumpPropMap("========WebHCat System.getProperties()========", System.getProperties()));
     return sb.toString();
   }
   public void startCleanup() {

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1640981&r1=1640980&r2=1640981&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Fri Nov 21 19:24:54 2014
@@ -18,6 +18,7 @@
  */
 package org.apache.hive.hcatalog.templeton.tool;
 
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,6 +35,8 @@ import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
+import java.util.StringTokenizer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -408,4 +411,34 @@ public class TempletonUtils {
     }
     return null;
   }
+  public static StringBuilder dumpPropMap(String header, Properties props) {
+    Map<String, String> map = new HashMap<String, String>();
+    for(Map.Entry<Object, Object> ent : props.entrySet()) {
+      map.put(ent.getKey().toString(), ent.getValue() == null ? null : ent.getValue().toString());
+    }
+    return dumpPropMap(header, map);
+  }
+  public static StringBuilder dumpPropMap(String header, Map<String, String> map) {
+    StringBuilder sb = new StringBuilder("START").append(header).append(":\n");
+    List<String> propKeys = new ArrayList<String>(map.keySet());
+    Collections.sort(propKeys);
+    for(String propKey : propKeys) {
+      if(propKey.toLowerCase().contains("path")) {
+        StringTokenizer st = new StringTokenizer(map.get(propKey), File.pathSeparator);
+        if(st.countTokens() > 1) {
+          sb.append(propKey).append("=\n");
+          while (st.hasMoreTokens()) {
+            sb.append("    ").append(st.nextToken()).append(File.pathSeparator).append('\n');
+          }
+        }
+        else {
+          sb.append(propKey).append('=').append(map.get(propKey)).append('\n');
+        }
+      }
+      else {
+        sb.append(propKey).append('=').append(map.get(propKey)).append('\n');
+      }
+    }
+    return sb.append("END").append(header).append('\n');
+  }
 }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java?rev=1640981&r1=1640980&r2=1640981&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java Fri Nov 21 19:24:54 2014
@@ -20,7 +20,6 @@ package org.apache.hive.hcatalog.templet
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
 
 import java.io.File;
 import java.io.IOException;
@@ -28,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.StringTokenizer;
 
 
 /**
@@ -61,16 +61,47 @@ final class TrivialExecService {
       pb.environment().remove(key);
     }
     pb.environment().putAll(environmentVariables);
-    logDebugInfo("Starting process with env:", pb.environment());
+    logDebugInfo("========Starting process with env:========", pb.environment());
+    printContentsOfDir(".");
     return pb.start();
   }
   private static void logDebugInfo(String msg, Map<String, String> props) {
-    LOG.info(msg);
-    List<String> keys = new ArrayList<String>();
-    keys.addAll(props.keySet());
-    Collections.sort(keys);
-    for(String key : keys) {
-      LOG.info(key + "=" + props.get(key));
-    }    
+    StringBuilder sb = TempletonUtils.dumpPropMap(msg, props);
+    LOG.info(sb.toString());
+    String sqoopHome = props.get("SQOOP_HOME");
+    if(TempletonUtils.isset(sqoopHome)) {
+      //this is helpful when Sqoop is installed on each node in the cluster to make sure
+      //relevant jars (JDBC in particular) are present on the node running the command
+      printContentsOfDir(sqoopHome + File.separator+ "lib");
+    }
+  }
+  /**
+   * Print files and directories in current directory. Will list files in the sub-directory (only 1 level deep)
+   * time honored tradition in WebHCat of borrowing from Oozie
+   */
+  private static void printContentsOfDir(String dir) {
+    File folder = new File(dir);
+    StringBuilder sb = new StringBuilder("Files in '").append(dir).append("' dir:").append(folder.getAbsolutePath()).append('\n');
+
+    File[] listOfFiles = folder.listFiles();
+    for (File fileName : listOfFiles) {
+      if (fileName.isFile()) {
+        sb.append("File: ").append(fileName.getName()).append('\n');
+      }
+      else if (fileName.isDirectory()) {
+        sb.append("Dir: ").append(fileName.getName()).append('\n');
+        File subDir = new File(fileName.getName());
+        File[] moreFiles = subDir.listFiles();
+        for (File subFileName : moreFiles) {
+          if (subFileName.isFile()) {
+            sb.append("--File: ").append(subFileName.getName()).append('\n');
+          }
+          else if (subFileName.isDirectory()) {
+            sb.append("--Dir: ").append(subFileName.getName()).append('\n');
+          }
+        }
+      }
+    }
+    LOG.info(sb.toString());
   }
 }