You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/10/16 23:47:24 UTC
svn commit: r464685 - in /lucene/hadoop/trunk: CHANGES.txt
src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java
src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Author: cutting
Date: Mon Oct 16 14:47:23 2006
New Revision: 464685
URL: http://svn.apache.org/viewvc?view=rev&rev=464685
Log:
HADOOP-477. Extend contrib/streaming to scan the PATH environment variable when resolving executable program names. Contributed by Dhruba Borthakur.
Added:
lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=464685&r1=464684&r2=464685
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Oct 16 14:47:23 2006
@@ -1,6 +1,13 @@
Hadoop Change Log
+Trunk (unreleased changes)
+
+ 1. HADOOP-477. Extend contrib/streaming to scan the PATH environment
+ variables when resolving executable program names.
+ (Dhruba Borthakur via cutting)
+
+
Release 0.7.1 - 2006-10-11
1. HADOOP-593. Fix a NullPointerException in the JobTracker.
Added: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java?view=auto&rev=464685
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java (added)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java Mon Oct 16 14:47:23 2006
@@ -0,0 +1,146 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.streaming;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Maps a relative pathname to an absolute pathname using the
+ * PATH enviroment.
+ * @author Dhruba Borthakur
+ */
+public class PathFinder
+{
+ String pathenv; // a string of pathnames
+ String pathSep; // the path seperator
+ String fileSep; // the file seperator in a directory
+
+ /**
+ * Construct a PathFinder object using the path from
+ * java.class.path
+ */
+ public PathFinder()
+ {
+ pathenv = System.getProperty("java.class.path");
+ pathSep = System.getProperty("path.separator");
+ fileSep = System.getProperty("file.separator");
+ }
+
+ /**
+ * Construct a PathFinder object using the path from
+ * the specified system environment variable.
+ */
+ public PathFinder(String envpath)
+ {
+ pathenv = System.getenv(envpath);
+ pathSep = System.getProperty("path.separator");
+ fileSep = System.getProperty("file.separator");
+ }
+
+ /**
+ * Appends the specified component to the path list
+ */
+ public void prependPathComponent(String str)
+ {
+ pathenv = str + pathSep + pathenv;
+ }
+
+ /**
+ * Returns the full path name of this file if it is listed in the
+ * path
+ */
+ public File getAbsolutePath(String filename)
+ {
+ if (pathenv == null || pathSep == null || fileSep == null)
+ {
+ return null;
+ }
+ int val = -1;
+ String classvalue = pathenv + pathSep;
+
+ while (((val = classvalue.indexOf(pathSep)) >= 0) &&
+ classvalue.length() > 0) {
+ //
+ // Extract each entry from the pathenv
+ //
+ String entry = classvalue.substring(0, val).trim();
+ File f = new File(entry);
+
+ try {
+ if (f.isDirectory()) {
+ //
+ // this entry in the pathenv is a directory.
+ // see if the required file is in this directory
+ //
+ f = new File(entry + fileSep + filename);
+ }
+ //
+ // see if the filename matches and we can read it
+ //
+ if (f.isFile() && f.canRead()) {
+ return f;
+ }
+ } catch (Exception exp){ }
+ classvalue = classvalue.substring(val+1).trim();
+ }
+ return null;
+ }
+
+ /**
+ * prints all environment variables for this process
+ */
+ private static void printEnvVariables() {
+ System.out.println("Environment Variables: ");
+ Map<String,String> map = System.getenv();
+ Set<String> keys = map.keySet();
+ Iterator iter = keys.iterator();
+ while(iter.hasNext()) {
+ String thiskey = (String)(iter.next());
+ String value = map.get(thiskey);
+ System.out.println(thiskey + " = " + value);
+ }
+ }
+
+ /**
+ * prints all system properties for this process
+ */
+ private static void printSystemProperties() {
+ System.out.println("System properties: ");
+ java.util.Properties p = System.getProperties();
+ java.util.Enumeration keys = p.keys();
+ while(keys.hasMoreElements()) {
+ String thiskey = (String)keys.nextElement();
+ String value = p.getProperty(thiskey);
+ System.out.println(thiskey + " = " + value);
+ }
+ }
+
+ public static void main(String args[]) throws IOException {
+
+ if (args.length < 1) {
+ System.out.println("Usage: java PathFinder <filename>");
+ System.exit(1);
+ }
+
+ PathFinder finder = new PathFinder("PATH");
+ File file = finder.getAbsolutePath(args[0]);
+ if (file != null) {
+ System.out.println("Full path name = " + file.getCanonicalPath());
+ }
+ }
+}
Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?view=diff&rev=464685&r1=464684&r2=464685
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java (original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java Mon Oct 16 14:47:23 2006
@@ -273,11 +273,23 @@
sideEffectOut_ = getURIOutputStream(sideEffectURI_, allowSocket);
}
+ //
// argvSplit[0]:
// An absolute path should be a preexisting valid path on all TaskTrackers
- // A relative path should match in the unjarred Job data
- // In this case, force an absolute path to make sure exec finds it.
- argvSplit[0] = new File(argvSplit[0]).getAbsolutePath();
+ // A relative path is converted into an absolute pathname by looking
+ // up the PATH env variable. If it still fails, look it up in the
+ // tasktracker's local working directory
+ //
+ if (!new File(argvSplit[0]).isAbsolute()) {
+ PathFinder finder = new PathFinder("PATH");
+ finder.prependPathComponent(".");
+ File f = finder.getAbsolutePath(argvSplit[0]);
+ if (f != null) {
+ argvSplit[0] = f.getAbsolutePath();
+ }
+ f = null;
+ }
+ System.out.println("XXX2 argvSplit[0] = " + argvSplit[0]);
logprintln("PipeMapRed exec " + Arrays.asList(argvSplit));
logprintln("sideEffectURI_=" + sideEffectURI_);