You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/02/01 04:11:39 UTC

svn commit: r617338 [5/5] - in /incubator/pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/ src/org/apache/pig/backend/datastorage/ src/org/apache/pig/backend/executionengine/ src/org/apache/pig/backend/hadoop/ src/org/apache/pig/backend/ha...

Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java Thu Jan 31 19:11:28 2008
@@ -18,6 +18,7 @@
 package org.apache.pig.impl.logicalLayer;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.pig.impl.logicalLayer.schema.TupleSchema;
 
@@ -26,25 +27,30 @@
 public class LOUnion extends LogicalOperator {
     private static final long serialVersionUID = 1L;
 
-    public LOUnion(List<LogicalOperator> inputsIn) {
-        super(inputsIn);
+    public LOUnion(Map<OperatorKey, LogicalOperator> opTable,
+                   String scope, 
+                   long id, 
+                   List<OperatorKey> inputsIn) {
+        super(opTable, scope, id, inputsIn);
     }
     
     @Override
     public String name() {
-        return "Union";
+        return "Union " + scope + "-" + id;
     }
 
     @Override
     public TupleSchema outputSchema() {
         if (schema == null) {
-            TupleSchema longest = getInputs().get(0).outputSchema();
+            TupleSchema longest = opTable.get(getInputs().get(0)).outputSchema();
             int current = 0;
-          for (LogicalOperator lo:getInputs()) {
-                if (lo != null && lo.outputSchema() != null
-                        && lo.outputSchema().numFields() > current) {
-                    longest = lo.outputSchema();
-                    current = longest.numFields();
+          for (OperatorKey opKey: getInputs()) {
+              LogicalOperator lo = opTable.get(opKey);
+              
+              if (lo != null && lo.outputSchema() != null && 
+                  lo.outputSchema().numFields() > current) {
+                  longest = lo.outputSchema();
+                  current = longest.numFields();
                 }
             }
             schema = longest.copy();
@@ -58,7 +64,7 @@
     public int getOutputType() {
         int outputType = FIXED;
         for (int i = 0; i < getInputs().size(); i++) {
-            switch (getInputs().get(i).getOutputType()) {
+            switch (opTable.get(getInputs().get(i)).getOutputType()) {
             case FIXED:
                 continue;
             case MONOTONE:

Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java Thu Jan 31 19:11:28 2008
@@ -20,7 +20,6 @@
 import java.util.List;
 import java.util.Iterator;
 
-import org.apache.pig.impl.physicalLayer.IntermedResult;
 
 /**
  * A visitor mechanism for navigating and operating on a tree of Logical
@@ -69,17 +68,6 @@
         basicVisit(u);
     }
         
-    /**
-     * Only LORead.visit() and subclass implementations of this function
-     * should ever call this method.
-     */
-    public void visitRead(LORead r) {
-        // I'm assuming LORead is always at the head of a list of LO operators.
-        IntermedResult ir = r.getReadFrom();
-        if (ir != null) {
-            ir.lp.getRoot().visit(this);
-        }
-    }
         
     /**
      * Only LOLoad.visit() and subclass implementations of this function
@@ -104,6 +92,11 @@
     public void visitSplit(LOSplit s) {
         basicVisit(s);
     }
+    
+    public void visitSplitOutput(LOSplitOutput s) {
+        basicVisit(s);
+    }
+    
         
     /**
      * Only LOStore.visit() and subclass implementations of this function
@@ -114,10 +107,11 @@
     }
 
     private void basicVisit(LogicalOperator lo) {
-        List<LogicalOperator> inputs = lo.getInputs();
-        Iterator<LogicalOperator> i = inputs.iterator();
+        List<OperatorKey> inputs = lo.getInputs();
+        Iterator<OperatorKey> i = inputs.iterator();
+        
         while (i.hasNext()) {
-            LogicalOperator input = i.next();
+            LogicalOperator input = lo.getOpTable().get(i.next());
             input.visit(this);
         }
     }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Thu Jan 31 19:11:28 2008
@@ -21,12 +21,14 @@
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.pig.impl.logicalLayer.schema.TupleSchema;
 
+import org.apache.pig.backend.executionengine.ExecScopedLogicalOperator;
 
 
-abstract public class LogicalOperator implements Serializable {
+abstract public class LogicalOperator implements Serializable, ExecScopedLogicalOperator {
     public String alias = null;
 
     public static final int FIXED = 1;
@@ -36,19 +38,64 @@
 
     protected int requestedParallelism = -1;
     protected TupleSchema schema = null;
-    protected List<LogicalOperator> inputs;
+    protected List<OperatorKey> inputs;
 
-    protected LogicalOperator() {
-        this.inputs = new ArrayList<LogicalOperator> ();
-    } protected LogicalOperator(List<LogicalOperator> inputs) {
+    protected Map<OperatorKey, LogicalOperator> opTable;
+
+    protected String scope;
+    protected long id;
+    
+    protected LogicalOperator(Map<OperatorKey, LogicalOperator> opTable,
+                              String scope, 
+                              long id) {
+        this.inputs = new ArrayList<OperatorKey> ();
+        this.opTable = opTable;
+        this.scope = scope;
+        this.id = id;
+
+        opTable.put(getOperatorKey(), this);
+    } 
+    
+    protected LogicalOperator(Map<OperatorKey, LogicalOperator> opTable, 
+                              String scope, 
+                              long id, 
+                              List<OperatorKey> inputs) {
+        this.opTable = opTable;
         this.inputs = inputs;
+        this.scope = scope;
+        this.id = id;
+        
+        opTable.put(getOperatorKey(), this);
     }
 
-    protected LogicalOperator(LogicalOperator input) {
-        this.inputs = new ArrayList<LogicalOperator> ();
+    protected LogicalOperator(Map<OperatorKey, LogicalOperator> opTable,
+                              String scope, 
+                              long id, 
+                              OperatorKey input) {
+        this.opTable = opTable;
+        this.inputs = new ArrayList<OperatorKey> ();
         inputs.add(input);
-    }
+        this.scope = scope;
+        this.id = id;
 
+        opTable.put(getOperatorKey(), this);
+    }
+    
+    @Override
+    public OperatorKey getOperatorKey() {
+        return new OperatorKey(scope, id);
+    }
+    
+    @Override
+    public String getScope() {
+        return this.scope;
+    }
+    
+    @Override 
+    public long getId() {
+        return this.id;
+    }
+    
     public String getAlias() {
         return alias;
     }
@@ -78,13 +125,17 @@
     public abstract TupleSchema outputSchema();
 
     public String name() {
-        return "ROOT";
+        return "ROOT " + scope + "-" + id;
     }
 
-    public List<LogicalOperator> getInputs() {
+    public List<OperatorKey> getInputs() {
         return inputs;
     }
 
+    public Map<OperatorKey, LogicalOperator> getOpTable() {
+        return opTable;
+    }
+    
     public String arguments() {
         return "";
     }
@@ -92,7 +143,7 @@
     public List<String> getFuncs() {
         List<String> funcs = new LinkedList<String>();
         for (int i = 0; i < inputs.size(); i++) {
-            funcs.addAll(inputs.get(i).getFuncs());
+            funcs.addAll(opTable.get(inputs.get(i)).getFuncs());
         }
         return funcs;
     }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Thu Jan 31 19:11:28 2008
@@ -20,34 +20,44 @@
 import java.io.Serializable;
 import java.util.LinkedList;
 import java.util.List;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Map;
 
 import org.apache.pig.impl.PigContext;
 
+import org.apache.pig.backend.executionengine.ExecLogicalPlan;
 
-
-
-public class LogicalPlan implements Serializable{
+public class LogicalPlan implements Serializable, ExecLogicalPlan{
 	private static final long serialVersionUID = 1L;
 
-	protected LogicalOperator root;
+	protected OperatorKey root;
+	protected Map<OperatorKey, LogicalOperator> opTable;
 	protected PigContext pigContext = null;
 	
 	protected String alias = null;
-
-	public LogicalOperator getRoot() {
+	
+	public OperatorKey getRoot() {
 		return root;
 	}
 
-	public LogicalPlan(LogicalOperator rootIn, PigContext pigContext) {
+	public LogicalPlan(OperatorKey rootIn,
+	                   Map<OperatorKey, LogicalOperator> opTable ,
+	                   PigContext pigContext) {
 		this.pigContext = pigContext;
-		root = rootIn;
-        alias = root.alias;
+		this.root = rootIn;
+		this.opTable = opTable;
+        alias = opTable.get(root).alias;
+	}
+	
+	public Map<OperatorKey, LogicalOperator> getOpTable() {
+	    return this.opTable;
 	}
 	
-    public void setRoot(LogicalOperator newRoot) {
+    public void setRoot(OperatorKey newRoot) {
         root = newRoot;
     }
-	
+
 	public PigContext getPigContext() {
 		return pigContext;
 	}
@@ -60,35 +70,41 @@
 		alias = newAlias;
 	}
 
-
 	public List<String> getFuncs() {
         if (root == null) return new LinkedList<String>();
-        else return root.getFuncs();
+        else return opTable.get(root).getFuncs();
     }
 	
 	// indentation for root is 0
 	@Override
 	public String toString() {		
 		StringBuffer sb = new StringBuffer();
-		sb.append(root.name() +"(" + root.arguments() +")\n");
+		sb.append(opTable.get(root).name() +"(" + opTable.get(root).arguments() +")\n");
 		sb.append(appendChildren(root, 1));
 		return sb.toString();
 	}
-	public String appendChildren(LogicalOperator parent, int indentation) {
+	public String appendChildren(OperatorKey parent, int indentation) {
 		StringBuffer sb = new StringBuffer();
-		List<LogicalOperator> children = parent.getInputs();
+		List<OperatorKey> children = opTable.get(parent).getInputs();
 		for(int i=0; i != children.size(); i++) {
 			for(int j=0; j != indentation; j++) {
 				sb.append("\t");
 			}
-			sb.append(children.get(i).name() + "(" + children.get(i).arguments()+ ")\n");
+			
+			sb.append(opTable.get(children.get(i)).name() + 
+			          "(" + opTable.get(children.get(i)).arguments()+ ")\n");
 			sb.append(appendChildren(children.get(i), indentation+1));
 		}
 		return sb.toString();
 	}
 	
 	public int getOutputType(){
-		return root.getOutputType();
+		return opTable.get(root).getOutputType();
 	}
 	
+	public void explain(OutputStream out) {
+	    LOVisitor lprinter = new LOPrinter(new PrintStream(out));
+	    
+	    opTable.get(root).visit(lprinter);
+	}
 }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java Thu Jan 31 19:11:28 2008
@@ -24,7 +24,6 @@
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.parser.ParseException;
 import org.apache.pig.impl.logicalLayer.parser.QueryParser;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
 
 
 /**
@@ -38,16 +37,13 @@
 		this.pigContext = pigContext;
 	}
 
-	public LogicalPlan parse(String query, Map<String, IntermedResult> aliases)
+	public LogicalPlan parse(String scope, 
+	                         String query, 
+	                         Map<String, LogicalPlan> aliases,
+	                         Map<OperatorKey, LogicalOperator> opTable)
 		throws IOException, ParseException {
-		/*
-		if(query.matches(".*select.*")) { 
-			query = sqlToPigLatin(query);
-			//System.err.println(query);
-		}
-		*/
 		ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());		
-		QueryParser parser = new QueryParser(in, pigContext, aliases);
+		QueryParser parser = new QueryParser(in, pigContext, scope, aliases, opTable);
 		return parser.Parse();		
 	}
 }

Added: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java?rev=617338&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java Thu Jan 31 19:11:28 2008
@@ -0,0 +1,49 @@
+package org.apache.pig.impl.logicalLayer;
+
+import java.io.Serializable;
+
+public class OperatorKey implements Serializable {
+    private static final long serialVersionUID = 1L;
+    
+    public String scope;
+    public long id;
+
+    public OperatorKey() {
+        this("", -1);
+    }
+    
+    public OperatorKey(String scope, long id) {
+        this.scope = scope;
+        this.id = id;
+    }
+    
+    @Override
+    public String toString() {
+        return scope + "-" + id;
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (! (obj instanceof OperatorKey)) {
+            return false;
+        }
+        else {
+            OperatorKey otherKey = (OperatorKey) obj;
+            return this.scope.equals(otherKey.scope) &&
+                   this.id == otherKey.id;
+        }
+    }
+    
+    @Override
+    public int hashCode() {
+        return this.toString().hashCode();
+    }
+    
+    public String getScope() {
+        return scope;
+    }
+    
+    public long getId() {
+        return id;
+    }
+}
\ No newline at end of file

Added: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java?rev=617338&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java Thu Jan 31 19:11:28 2008
@@ -0,0 +1,32 @@
+package org.apache.pig.impl.logicalLayer.parser;
+
+import java.util.Map;
+import java.util.HashMap;
+
+public class NodeIdGenerator {
+
+    private Map<String, Long> scopeToIdMap;
+    private static NodeIdGenerator theGenerator = new NodeIdGenerator();
+    
+    private NodeIdGenerator() {
+        scopeToIdMap = new HashMap<String, Long>();
+    }
+    
+    public static NodeIdGenerator getGenerator() {
+        return theGenerator;
+    }
+    
+    public long getNextNodeId(String scope) {
+        Long val = scopeToIdMap.get(scope);
+        
+        long nextId = 0;
+        
+        if (val != null) {
+            nextId = val.longValue();
+        }
+
+        scopeToIdMap.put(scope, nextId + 1);
+        
+        return nextId;
+    }
+}

Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Thu Jan 31 19:11:28 2008
@@ -40,19 +40,33 @@
 import org.apache.pig.data.*;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.PigServer.ExecType;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
 import org.apache.pig.impl.io.*;
 import org.apache.pig.builtin.*;
 import org.apache.pig.impl.builtin.*;
+import org.apache.pig.impl.logicalLayer.LogicalPlan;
 
 public class QueryParser {
 	private PigContext pigContext;
-	private Map<String, IntermedResult> aliases;
+	private Map<String, LogicalPlan> aliases;
+	private Map<OperatorKey, LogicalOperator> opTable;
+	private String scope;
+	private NodeIdGenerator nodeIdGen;
+
+	private long getNextId() {
+		return nodeIdGen.getNextNodeId(scope);
+	}
 
-	public QueryParser(InputStream in, PigContext pigContext, Map aliases) {
+	public QueryParser(InputStream in, 
+					   PigContext pigContext, 
+					   String scope, 
+					   Map<String, LogicalPlan> aliases,
+					   Map<OperatorKey, LogicalOperator> opTable) {
 		this(in);
 		this.pigContext = pigContext;
 		this.aliases = aliases;
+		this.opTable = opTable;
+		this.scope = scope;
+		this.nodeIdGen = NodeIdGenerator.getGenerator();
 	}
 	
 	public class EvalSpecAndSchema{
@@ -61,10 +75,43 @@
 	}
 	
 	public class CogroupInput {
-		public LogicalOperator op;
+		public OperatorKey op;
 		public EvalSpec spec;
 	}
 	    
+    private static String removeQuotes(String str) {
+        if (str.startsWith("\'") && str.endsWith("\'"))
+            return str.substring(1, str.length() - 1);
+        else
+            return str;
+    }
+
+    public static LogicalPlan generateStorePlan(Map<OperatorKey, LogicalOperator> opTable,
+    										    String scope,
+                                                LogicalPlan readFrom,
+                                                String fileName,
+                                                String func,
+                                                PigContext pigContext) throws IOException {
+        if (func == null) {
+            func = PigStorage.class.getName();
+        }
+
+        fileName = removeQuotes(fileName);
+
+        long storeNodeId = NodeIdGenerator.getGenerator().getNextNodeId(scope);
+
+		LogicalOperator root = new LOStore(opTable,
+										   scope,
+                                           storeNodeId,
+                                           readFrom.getRoot(),
+                                           new FileSpec(fileName, func),
+                                           false);
+                                           
+        LogicalPlan storePlan = new LogicalPlan(root.getOperatorKey(), opTable, pigContext);
+
+        return storePlan;
+    }
+
     static String unquote(String s) {
 		return s.substring(1, s.length()-1);
 	}
@@ -74,20 +121,11 @@
 	}
 	
 	
-	LogicalOperator makeLORead(String alias) throws ParseException {
-		if (!aliases.containsKey(alias)) throw new ParseException("Unrecognized alias: " + alias);
-	
-		LORead readOp = new LORead(aliases.get(alias));
-		readOp.setAlias(alias);
-		return readOp;
-	}
-	
-
 	
 	String massageFilename(String filename, PigContext pigContext) throws IOException, ParseException {
 		if (pigContext.getExecType() != ExecType.LOCAL) {
 			if (filename.startsWith(FileLocalizer.LOCAL_PREFIX)) {
-				filename = FileLocalizer.hadoopify(filename, pigContext);
+					filename = FileLocalizer.hadoopify(filename, pigContext);
 			} 
 			else
 			{
@@ -104,7 +142,7 @@
 	LogicalOperator parseCogroup(ArrayList<CogroupInput> gis) throws ParseException{
 		int n = gis.size();
 		
-		List<LogicalOperator> los = new ArrayList<LogicalOperator>();
+		List<OperatorKey> los = new ArrayList<OperatorKey>();
 		ArrayList<EvalSpec> specs = new ArrayList<EvalSpec>();
 		
 		for (int i = 0; i < n ; i++){
@@ -114,11 +152,11 @@
 			specs.add(gi.spec);
 		}
 		
-		return new LOCogroup(los, specs);
+		return new LOCogroup(opTable, scope, getNextId(), los, specs);
 	}
 			
 	
-	LogicalOperator rewriteCross(ArrayList<LogicalOperator> inputs) throws IOException, ParseException{
+	LogicalOperator rewriteCross(ArrayList<OperatorKey> inputs) throws IOException, ParseException{
 		ArrayList<CogroupInput> gis = new ArrayList<CogroupInput>();
 		int n = inputs.size();
 		
@@ -140,23 +178,23 @@
 		return rewriteJoin(gis); 
 	}
 
-	LogicalOperator rewriteDistinct(LogicalOperator input){
+	LogicalOperator rewriteDistinct(OperatorKey input){
 		//First group the input on *
 		
-		ArrayList<LogicalOperator> inputs = new ArrayList<LogicalOperator>();
+		ArrayList<OperatorKey> inputs = new ArrayList<OperatorKey>();
 		inputs.add(input);
 		
 		ArrayList<EvalSpec> groupSpecs = new ArrayList<EvalSpec>();
 			
 		groupSpecs.add(new GenerateSpec(new StarSpec()).getGroupBySpec());
 		
-		LogicalOperator groupedInput = new LOCogroup(inputs, groupSpecs);
+		LogicalOperator groupedInput = new LOCogroup(opTable, scope, getNextId(), inputs, groupSpecs);
 		
 		//then generate the flattened group
 		EvalSpec projectSpec = new ProjectSpec(0);
 		projectSpec.setFlatten(true);
 		
-		return new LOEval(groupedInput, new GenerateSpec(projectSpec));
+		return new LOEval(opTable, scope, getNextId(), groupedInput.getOperatorKey()	, new GenerateSpec(projectSpec));
 	}	
 		
 		
@@ -172,7 +210,7 @@
 			flattenedColumns.add(column);
 		}
 		
-		return new LOEval(parseCogroup(gis),new GenerateSpec(flattenedColumns));
+		return new LOEval(opTable, scope, getNextId(), parseCogroup(gis).getOperatorKey(),new GenerateSpec(flattenedColumns));
 	}
 		
 	void assertAtomic(EvalSpec spec, boolean desiredAtomic) throws ParseException{
@@ -199,10 +237,10 @@
 		return spec.addSpec(successor);
 	}
 	
-	 void addSplitOutput(IntermedResult intermedResult, LOSplit splitOp, String alias, Cond cond, int index){
+	 void addSplitOutput(LogicalPlan lp, LOSplit splitOp, String alias, Cond cond, int index){
 		splitOp.addCond(cond);
-		LORead readOp = new LORead(intermedResult, index);
-		aliases.put(alias, new IntermedResult(new LogicalPlan(readOp, pigContext), pigContext));
+		LOSplitOutput splitOut = new LOSplitOutput(opTable, scope, getNextId(), lp.getRoot(), index);
+		aliases.put(alias, new LogicalPlan(splitOut.getOperatorKey(), opTable, pigContext));
 	 }
 
 }
@@ -290,27 +328,27 @@
 |	(root = Expr() ";")
 |	(<SPLIT> root = SplitClause() ";")
 	)
-	{ return new LogicalPlan(root, pigContext); }
+	{ return new LogicalPlan(root.getOperatorKey(), opTable, pigContext); }
 }
 
 LogicalOperator SplitClause():
-{LogicalOperator input; Cond cond; Token alias; LOSplit splitOp; IntermedResult intermedResult; int i=0;}
+{LogicalOperator input; Cond cond; Token alias; LOSplit splitOp; LogicalPlan lp; int i=0;}
 {
 	(
 	input = NestedExpr() <INTO> 
 	{
-		splitOp = new LOSplit(input);
-		intermedResult = new IntermedResult(new LogicalPlan(splitOp, pigContext), pigContext);
+		splitOp = new LOSplit(opTable, scope, getNextId(), input.getOperatorKey());
+		lp = new LogicalPlan(splitOp.getOperatorKey(), opTable, pigContext);
 	}
 	alias = <IDENTIFIER> <IF> cond = PCond(input.outputSchema(),null) 
 	{
-		addSplitOutput(intermedResult, splitOp, alias.image, cond, i);
+		addSplitOutput(lp, splitOp, alias.image, cond, i);
 		i++;
 	}
 	(
 	"," alias = <IDENTIFIER> <IF> cond = PCond(input.outputSchema(),null)
 	{
-		addSplitOutput(intermedResult, splitOp, alias.image, cond, i);
+		addSplitOutput(lp, splitOp, alias.image, cond, i);
 		i++;
 	}
 	)+
@@ -343,8 +381,16 @@
 {
 	t1 = <IDENTIFIER> 
 	{
-		op = makeLORead(t1.image);
-		return op;
+		LogicalOperator aliasOp;
+		String alias = t1.image;
+		
+		aliasOp = opTable.get(aliases.get(alias).getRoot());
+		
+		if (aliasOp == null) {
+			throw new ParseException("Unrecognized alias " + alias);
+		}
+		
+		return aliasOp;
 	}
 }
 
@@ -360,7 +406,7 @@
 |	((<GROUP> | <COGROUP>) op = CogroupClause())
 |	(<FILTER> op = FilterClause())
 |   (<ORDER> op = OrderClause())
-|	(<DISTINCT> op = NestedExpr() {op = rewriteDistinct(op);})
+|	(<DISTINCT> op = NestedExpr() {op = rewriteDistinct(op.getOperatorKey());})
 |	(<CROSS> op = CrossClause())
 |   (<JOIN> op = JoinClause())
 |	(<UNION> op = UnionClause())
@@ -389,7 +435,7 @@
 			funcSpec += continuous ? "('\t','\n','0')" : "()";
 		}
 		 
-		lo = new LOLoad(new FileSpec(massageFilename(filename, pigContext), funcSpec));	
+		lo = new LOLoad(opTable, scope, getNextId(), new FileSpec(massageFilename(filename, pigContext), funcSpec));	
 		if (continuous)
 			lo.setOutputType(LogicalOperator.MONOTONE);
 		return lo;
@@ -420,7 +466,7 @@
 	input = NestedExpr()	
 	 <BY> cond = PCond(input.outputSchema(),null)
 	{
-		return new LOEval(input, new FilterSpec(cond));
+		return new LOEval(opTable, scope, getNextId(), input.getOperatorKey(), new FilterSpec(cond));
 	}
 }
 
@@ -491,27 +537,34 @@
 }
 
 
-CogroupInput GroupItem() : {LogicalOperator op; GenerateSpec gs; EvalSpec es; CogroupInput cogroupInput = new CogroupInput();}
+CogroupInput GroupItem() : {LogicalOperator op; GenerateSpec gs; EvalSpec es; LogicalOperator cgOp; EvalSpec cgSpec;}
 {
 	(
-	cogroupInput.op = NestedExpr()
+	cgOp = NestedExpr()
 	(
 	( <BY> 	
 	( 
-	LOOKAHEAD ( "(" FlattenedGenerateItemList(cogroupInput.op.outputSchema()) ")" )
-	( "(" gs = FlattenedGenerateItemList(cogroupInput.op.outputSchema(), null) ")" )
-|	(es = FlattenedGenerateItem(cogroupInput.op.outputSchema(), null) {gs = new GenerateSpec(es);})
+	LOOKAHEAD ( "(" FlattenedGenerateItemList(cgOp.outputSchema()) ")" )
+	( "(" gs = FlattenedGenerateItemList(cgOp.outputSchema(), null) ")" )
+|	(es = FlattenedGenerateItem(cgOp.outputSchema(), null) {gs = new GenerateSpec(es);})
 	)
 	)	
 |	<ALL> {gs = new GenerateSpec(new ConstSpec("all"));}
 |	<ANY> {gs = new GenerateSpec(new FuncEvalSpec(pigContext, GFAny.class.getName(), null));}
 	)
 	{ 
-		cogroupInput.spec = gs.getGroupBySpec();
+		cgSpec = gs.getGroupBySpec();
 	}
-	[<INNER> {cogroupInput.spec.setInner(true);} | <OUTER>]
+	[<INNER> {cgSpec.setInner(true);} | <OUTER>]
 	)
-	{return cogroupInput;}
+	{
+		CogroupInput cogroupInput = new CogroupInput(); 
+
+		cogroupInput.spec = cgSpec;
+		cogroupInput.op = cgOp.getOperatorKey();
+		
+		return cogroupInput;
+    }
 }
 
 
@@ -543,16 +596,16 @@
 
 	)
 	{
-		return new LOSort(op, sortSpec);
+		return new LOSort(opTable, scope, getNextId(), op.getOperatorKey(), sortSpec);
 	}
 }
 	
 
-LogicalOperator CrossClause() : {LogicalOperator op; ArrayList<LogicalOperator> inputs = new ArrayList<LogicalOperator>();}
+LogicalOperator CrossClause() : {LogicalOperator op; ArrayList<OperatorKey> inputs = new ArrayList<OperatorKey>();}
 {
 	(
-	op = NestedExpr() { inputs.add(op); }
-	("," op = NestedExpr() { inputs.add(op); })*
+	op = NestedExpr() { inputs.add(op.getOperatorKey()); }
+	("," op = NestedExpr() { inputs.add(op.getOperatorKey()); })*
 	)
 	{return rewriteCross(inputs);}
 }
@@ -564,11 +617,11 @@
 	{return rewriteJoin(gis);}
 }
 
-LogicalOperator UnionClause() : {LogicalOperator op; ArrayList<LogicalOperator> inputs = new ArrayList<LogicalOperator>();}
+LogicalOperator UnionClause() : {LogicalOperator op; ArrayList<OperatorKey> inputs = new ArrayList<OperatorKey>();}
 {
-	(op = NestedExpr() { inputs.add(op); }
-	("," op = NestedExpr() { inputs.add(op); })*)
-	{return new LOUnion(inputs);}
+	(op = NestedExpr() { inputs.add(op.getOperatorKey()); }
+	("," op = NestedExpr() { inputs.add(op.getOperatorKey()); })*)
+	{return new LOUnion(opTable, scope, getNextId(), inputs);}
 }
 
 
@@ -579,7 +632,7 @@
 	spec = NestedBlock(input.outputSchema())
 	)
 	{
-		op = new LOEval(input, spec);
+		op = new LOEval(opTable, scope, getNextId(), input.getOperatorKey(), spec);
 		return op;
 	}
 }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java Thu Jan 31 19:11:28 2008
@@ -19,36 +19,55 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.Map;
 
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
 import org.apache.pig.impl.util.LineageTracer;
+import org.apache.pig.backend.executionengine.ExecPhysicalOperator;
+import org.apache.pig.backend.local.executionengine.POVisitor;
 
+public abstract class PhysicalOperator implements Serializable, ExecPhysicalOperator {
 
-abstract public class PhysicalOperator implements Serializable {
-
-    public PhysicalOperator[] inputs = null;
-    int outputType;
+    public Map<OperatorKey, ExecPhysicalOperator> opTable;
+    public OperatorKey[] inputs = null;
+    public int outputType;
     protected LineageTracer lineageTracer = null;
+    protected String scope;
+    protected long id;
     
-    /**
-     * 
-     * This constructor should go away when we eventually implement CQ even for the mapreduce exec type 
-     */
-    public PhysicalOperator(){
-    	outputType = LogicalOperator.FIXED;
+    public PhysicalOperator(String scope, 
+                            long id,
+                            Map<OperatorKey, ExecPhysicalOperator> opTable, 
+                            int outputType){
+        this.scope = scope;
+        this.id = id;
+        this.opTable = opTable;
+    	this.outputType = outputType;
+    	
+    	opTable.put(getOperatorKey(), this);
     }
-   
     
-    public PhysicalOperator(int outputType){
-    	this.outputType = outputType;
+    public OperatorKey getOperatorKey() {
+        return new OperatorKey(scope, id);
+    }
+    
+    @Override
+    public String getScope() {
+        return this.scope;
+    }
+    
+    @Override
+    public long getId() {
+        return this.id;
     }
     
-    public boolean open(boolean continueFromLast) throws IOException {
+    public boolean open() throws IOException {
         // call open() on all inputs
         if (inputs != null) {
             for (int i = 0; i < inputs.length; i++) {
-                if (!inputs[i].open(continueFromLast))
+                if (! ((PhysicalOperator)opTable.get(inputs[i])).open())
                     return false;
             }
         }
@@ -61,7 +80,7 @@
         // call close() on all inputs
         if (inputs != null)
             for (int i = 0; i < inputs.length; i++)
-                inputs[i].close();
+                ((PhysicalOperator)opTable.get(inputs[i])).close();
     }
     
     public void setLineageTracer(LineageTracer lineageTracer) {
@@ -72,5 +91,5 @@
     	return outputType;
     }
 
-    public abstract void visit(POVisitor v);
+    public abstract void visit(POVisitor v, String prfix);
 }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java Thu Jan 31 19:11:28 2008
@@ -26,14 +26,7 @@
 
 
 public abstract class PlanCompiler {
-	PigContext pigContext;
-	protected PlanCompiler(PigContext pigContext) {
-		this.pigContext = pigContext;
-	}
+
     public abstract PhysicalOperator compile(LogicalOperator lo, Map queryResults) throws IOException;
 
-    // various helper methods for map-reduce and hadoop compilation:
-    public static String getTempFile(PigContext pigContext) throws IOException {
-        return FileLocalizer.getTemporaryPath(null, pigContext).toString();
-    }
 }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Thu Jan 31 19:11:28 2008
@@ -38,8 +38,8 @@
 import java.util.jar.JarOutputStream;
 import java.util.zip.ZipEntry;
 
+import org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigMapReduce;
 import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.mapreduceExec.PigMapReduce;
 
 
 public class JarManager {

Modified: incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java Thu Jan 31 19:11:28 2008
@@ -22,7 +22,7 @@
 import org.apache.pig.PigServer;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.tools.grunt.GruntParser;
-
+import org.apache.pig.backend.executionengine.ExecException;
 
 
 public class Grunt 
@@ -31,7 +31,7 @@
 	PigServer pig;
 	GruntParser parser;	
 
-	public Grunt(BufferedReader in, PigContext pigContext) 
+	public Grunt(BufferedReader in, PigContext pigContext) throws ExecException
 	{
 		this.in = in;
 		this.pig = new PigServer(pigContext);

Modified: incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Thu Jan 31 19:11:28 2008
@@ -4,20 +4,27 @@
 import java.io.InputStream;
 import java.io.Reader;
 import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.RunningJob;
+
 import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.tools.pigscript.parser.ParseException;
 import org.apache.pig.tools.pigscript.parser.PigScriptParser;
 import org.apache.pig.tools.pigscript.parser.PigScriptParserTokenManager;
+import org.apache.pig.backend.datastorage.DataStorage;
+import org.apache.pig.backend.datastorage.DataStorageException;
+import org.apache.pig.backend.datastorage.ElementDescriptor;
+import org.apache.pig.backend.datastorage.ContainerDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
 
 public class GruntParser extends PigScriptParser {
 
@@ -72,9 +79,18 @@
 	public void setParams(PigServer pigServer)
 	{
 		mPigServer = pigServer;
+		
 		mDfs = mPigServer.getPigContext().getDfs();
+		mLfs = mPigServer.getPigContext().getLfs();
 		mConf = mPigServer.getPigContext().getConf();
-		mJobClient = mPigServer.getPigContext().getJobClient();
+		
+		// TODO: this violates the abstraction layer decoupling between
+		// front end and back end and needs to be changed.
+		// Right now I am not clear on how the Job Id comes from to tell
+		// the back end to kill a given job (mJobClient is used only in 
+		// processKill)
+		//
+		mJobClient = ((HExecutionEngine)(mPigServer.getPigContext().getExecutionEngine())).getJobClient();
 	}
 
 	public void prompt()
@@ -137,50 +153,74 @@
 		
 	protected void processCat(String path) throws IOException
 	{
-		byte buffer[] = new byte[65536];
-		Path dfsPath = new Path(path);
-		int rc;
-		
-		if (!mDfs.exists(dfsPath))
-			throw new IOException("Directory " + path + " does not exist.");
-
-		if (mDfs.getFileStatus(dfsPath).isDir()) 
-		{
-			FileStatus fileStat[] = mDfs.listStatus(dfsPath);
-			for (int j = 0; j < fileStat.length; j++)
-			{
-				Path curPath = fileStat[j].getPath();
-				if (!mDfs.isFile(curPath)) continue;
-				FSDataInputStream is = mDfs.open(curPath);
-				while ((rc = is.read(buffer)) > 0)
+		try {
+			byte buffer[] = new byte[65536];
+			ElementDescriptor dfsPath = mDfs.asElement(path);
+			int rc;
+			
+			if (!dfsPath.exists())
+				throw new IOException("Directory " + path + " does not exist.");
+	
+			if (mDfs.isContainer(path)) {
+				ContainerDescriptor dfsDir = (ContainerDescriptor) dfsPath;
+				Iterator<ElementDescriptor> paths = dfsDir.iterator();
+				
+				while (paths.hasNext()) {
+					ElementDescriptor curElem = paths.next();
+					
+					if (mDfs.isContainer(curElem.toString())) {
+						continue;
+					}
+					
+					InputStream is = curElem.open();
+					while ((rc = is.read(buffer)) > 0) {
+						System.out.write(buffer, 0, rc);
+					}
+					is.close();				
+				}
+			}
+			else {
+				InputStream is = dfsPath.open();
+				while ((rc = is.read(buffer)) > 0) {
 					System.out.write(buffer, 0, rc);
-				is.close();
+				}
+				is.close();			
 			}
-		} 
-		else 
-		{
-			FSDataInputStream is = mDfs.open(dfsPath);
-			while ((rc = is.read(buffer)) > 0)
-				System.out.write(buffer, 0, rc);
-			is.close();
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to Cat: " + path, e);
 		}
 	}
 
 	protected void processCD(String path) throws IOException
 	{	
-		if (path == null)
-			mDfs.setWorkingDirectory(new Path("/user/" + System.getProperty("user.name")));
-		else
-		{
-			Path dfsDir = new Path(path);
-
-			if (!mDfs.exists(dfsDir))
-				throw new IOException("Directory " + path + " does not exist.");
-
-			if (!mDfs.getFileStatus(dfsDir).isDir())
-				throw new IOException(path + " is not a directory.");
+		ContainerDescriptor container;
 
-			mDfs.setWorkingDirectory(dfsDir);
+		try {
+			if (path == null) {
+				container = mDfs.asContainer("/user/" + System.getProperty("user.name"));
+				mDfs.setActiveContainer(container);
+			}
+			else
+			{
+				container = mDfs.asContainer(path);
+	
+				if (!container.exists()) {
+					throw new IOException("Directory " + path + " does not exist.");
+				}
+				
+				if (!mDfs.isContainer(path)) {
+					throw new IOException(path + " is not a directory.");
+				}
+				
+				mDfs.setActiveContainer(container);
+			}
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to change working directory to " + 
+								  ((path == null) ? ("/user/" + System.getProperty("user.name")) 
+									       	      : (path)),
+								  e);
 		}
 	}
 
@@ -208,29 +248,57 @@
 
 	protected void processLS(String path) throws IOException
 	{
-		Path dir;
-		if(path == null)
-                	dir = mDfs.getWorkingDirectory();
-		else
-		{
-			dir = new Path(path);
-			if (!mDfs.exists(dir))
-				throw new IOException("File or directory " + path + " does not exist.");
-		}
+		try {
+			ElementDescriptor pathDescriptor;
+			
+			if (path == null) {
+				pathDescriptor = mDfs.getActiveContainer();
+			}
+			else {
+				pathDescriptor = mDfs.asElement(path);
+			}
 
-		FileStatus fileStat[] = mDfs.listStatus(dir);
-		for (int j = 0; j < fileStat.length; j++)
-		{
-            if (fileStat[j].isDir())
-           		System.out.println(fileStat[j].getPath() + "\t<dir>");
-			else
-				System.out.println(fileStat[j].getPath() + "<r " + fileStat[j].getReplication() + ">\t" + fileStat[j].getLen());
-                }
+			if (!pathDescriptor.exists()) {
+				throw new IOException("File or directory " + path + " does not exist.");				
+			}
+			
+			if (mDfs.isContainer(pathDescriptor.toString())) {
+				ContainerDescriptor container = (ContainerDescriptor) pathDescriptor;
+				Iterator<ElementDescriptor> elems = container.iterator();
+				
+				while (elems.hasNext()) {
+					ElementDescriptor curElem = elems.next();
+					
+					if (mDfs.isContainer(curElem.toString())) {
+		           		System.out.println(curElem.toString() + "\t<dir>");						
+					}
+					else {
+						Properties config = curElem.getConfiguration();
+						Map<String, Object> stats = curElem.getStatistics();
+						
+						String strReplication = config.getProperty(ElementDescriptor.BLOCK_REPLICATION_KEY);
+						String strLen = (String) stats.get(ElementDescriptor.LENGTH_KEY);
+
+						System.out.println(curElem.toString() + "<r " + strReplication + ">\t" + strLen);
+					}
+				}
+			}
+			else {
+				Map<String, Object> properties = pathDescriptor.getStatistics();
+				String strReplication = (String) properties.get(DataStorage.DEFAULT_REPLICATION_FACTOR_KEY);
+				String strLen = (String) properties.get(DataStorage.USED_BYTES_KEY);
+
+				System.out.println(pathDescriptor.toString() + "<r " + strReplication + ">\t" + strLen);				
+			}
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to LS on " + path, e);
+		}
 	}
 	
 	protected void processPWD() throws IOException 
 	{
-		System.out.println(mDfs.getWorkingDirectory());
+		System.out.println(mDfs.getActiveContainer().toString());
 	}
 
 	protected void printHelp() 
@@ -253,32 +321,70 @@
 
 	protected void processMove(String src, String dst) throws IOException
 	{
-		Path srcPath = new Path(src);
-		Path dstPath = new Path(dst);
-		if (!mDfs.exists(srcPath))
-			throw new IOException("File or directory " + src + " does not exist.");
-
-		{mDfs.rename(srcPath, dstPath);}
+		try {
+			ElementDescriptor srcPath = mDfs.asElement(src);
+			ElementDescriptor dstPath = mDfs.asElement(dst);
+			
+			if (!srcPath.exists()) {
+				throw new IOException("File or directory " + src + " does not exist.");				
+			}
+			
+			srcPath.rename(dstPath);
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to move " + src + " to " + dst, e);
+		}
 	}
 	
 	protected void processCopy(String src, String dst) throws IOException
 	{
-        FileUtil.copy(mDfs, new Path(src), mDfs, new Path(dst), false, mConf);
+		try {
+			ElementDescriptor srcPath = mDfs.asElement(src);
+			ElementDescriptor dstPath = mDfs.asElement(dst);
+			
+			srcPath.copy(dstPath, mConf, false);
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to copy " + src + " to " + dst, e);
+		}
 	}
 	
 	protected void processCopyToLocal(String src, String dst) throws IOException
 	{
-		mDfs.copyToLocalFile(new Path(src), new Path(dst));
+		try {
+			ElementDescriptor srcPath = mDfs.asElement(src);
+			ElementDescriptor dstPath = mLfs.asElement(dst);
+			
+			srcPath.copy(dstPath, false);
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to copy " + src + "to (locally) " + dst, e);
+		}
 	}
 
 	protected void processCopyFromLocal(String src, String dst) throws IOException
 	{
-        mDfs.copyFromLocalFile(new Path(src), new Path(dst));
+		try {
+			ElementDescriptor srcPath = mLfs.asElement(src);
+			ElementDescriptor dstPath = mDfs.asElement(dst);
+			
+			srcPath.copy(dstPath, false);
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to copy (loally) " + src + "to " + dst, e);
+		}
 	}
 	
 	protected void processMkdir(String dir) throws IOException
 	{
-		mDfs.mkdirs(new Path(dir));
+		try {
+			ContainerDescriptor dirDescriptor = mDfs.asContainer(dir);
+			
+			dirDescriptor.create();
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to create dir: " + dir, e);
+		}
 	}
 	
 	protected void processPig(String cmd) throws IOException
@@ -291,16 +397,24 @@
 
 	protected void processRemove(String path) throws IOException
 	{
-		Path dfsPath = new Path(path);
-		if (!mDfs.exists(dfsPath))
-			throw new IOException("File or directory " + path + " does not exist.");
-
-		mDfs.delete(dfsPath);
+		try {
+			ElementDescriptor dfsPath = mDfs.asElement(path);
+		
+			if (!dfsPath.exists()) {
+				throw new IOException("File or directory " + path + " does not exist.");			
+			}
+			
+			dfsPath.delete();
+		}
+		catch (DataStorageException e) {
+			throw new IOException("Failed to get descriptor for " + path, e);
+		}
 	}
 
 	private PigServer mPigServer;
-	private FileSystem mDfs;
-	private Configuration mConf;
+	private DataStorage mDfs;
+	private DataStorage mLfs;
+	private Properties mConf;
 	private JobClient mJobClient;
 	private boolean mDone;
 

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java Thu Jan 31 19:11:28 2008
@@ -33,38 +33,8 @@
 public class TestAlgebraicEval extends TestCase {
     
 	private String initString = "mapreduce";
-    
-    @Test
-    public void testGroupCountWithMultipleFields() throws Exception {
-        int LOOP_COUNT = 1024;
-        PigServer pig = new PigServer(initString);
-        File tmpFile = File.createTempFile("test", "txt");
-        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
-        for(int i = 0; i < LOOP_COUNT; i++) {
-            for(int j=0; j< LOOP_COUNT; j++) {
-                ps.println(i + "\t" + i + "\t" + j%2);
-            }
-        }
-        ps.close();
-        String query = "myid = foreach (group (load 'file:" + tmpFile + "') all) generate group, COUNT($1) ;";
-        System.out.println(query);
-        pig.registerQuery(" a = group (load 'file:" + tmpFile + "') by ($0,$1);");
-        pig.registerQuery("b = foreach a generate flatten(group), SUM($1.$2);");
-        Iterator<Tuple> it = pig.openIterator("b");
-        tmpFile.delete();
-        int count = 0;
-        while(it.hasNext()){
-            int sum = it.next().getAtomField(2).numval().intValue();
-            assertEquals(LOOP_COUNT/2, sum);
-            count++;
-        }
-        assertEquals(count, LOOP_COUNT);
-    }
-    
-    
-    
     @Test
-    public void testSimpleCount() throws Exception {
+    public void testSimpleCount() throws Throwable {
         int LOOP_COUNT = 1024;
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
@@ -84,7 +54,7 @@
     }
 
     @Test
-    public void testGroupCount() throws Exception {
+    public void testGroupCount() throws Throwable {
         int LOOP_COUNT = 1024;
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
@@ -103,10 +73,8 @@
         assertEquals(count, (double)LOOP_COUNT);
     }
     
-    
-    
     @Test
-    public void testGroupReorderCount() throws Exception {
+    public void testGroupReorderCount() throws Throwable {
         int LOOP_COUNT = 1024;
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
@@ -128,7 +96,7 @@
 
 
     @Test
-    public void testGroupUniqueColumnCount() throws Exception {
+    public void testGroupUniqueColumnCount() throws Throwable {
         int LOOP_COUNT = 1024;
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
@@ -156,7 +124,7 @@
     }
 
     @Test
-    public void testGroupDuplicateColumnCount() throws Exception {
+    public void testGroupDuplicateColumnCount() throws Throwable {
         int LOOP_COUNT = 1024;
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java Thu Jan 31 19:11:28 2008
@@ -456,7 +456,7 @@
     }
     
     @Test
-    public void testShellFuncSingle() throws Exception {
+    public void testShellFuncSingle() throws Throwable {
     	//ShellBagEvalFunc func = new ShellBagEvalFunc("tr o 0");
     	PigServer pig = new PigServer(initString);
     	
@@ -484,7 +484,7 @@
     }
     
     @Test
-    public void testShellFuncMultiple() throws Exception {
+    public void testShellFuncMultiple() throws Throwable {
 
     	PigServer pig = new PigServer(initString);
     	final int numTimes = 100;

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java Thu Jan 31 19:11:28 2008
@@ -65,7 +65,7 @@
     }
     
     @Test
-    public void testCompressed1() throws Exception {
+    public void testCompressed1() throws Throwable {
         PigServer pig = new PigServer("mapreduce");
         pig.registerQuery("A = foreach (cogroup (load 'file:"+gzFile+"') by $1, (load 'file:"+datFile + "') by $1) generate flatten( " + DIFF.class.getName() + "($1.$1,$2.$1)) ;");
         Iterator it = pig.openIterator("A");

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java Thu Jan 31 19:11:28 2008
@@ -40,6 +40,8 @@
 import org.apache.pig.data.*;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.io.PigFile;
+
+import org.apache.pig.backend.executionengine.ExecException;
 
 import junit.framework.TestCase;
 
@@ -67,9 +69,9 @@
 		pw.close();
 		return f;
 	}
-	
+
 	@Test
-	public void testFunctionInsideFunction() throws Exception{
+	public void testFunctionInsideFunction() throws Throwable {
 		PigServer pigServer = new PigServer(initString);
 		
 		File f1 = createFile(new String[]{"a:1","b:1","a:1"});
@@ -85,7 +87,7 @@
 	}
 	
 	@Test
-	public void testJoin() throws Exception{
+	public void testJoin() throws Throwable {
 		PigServer pigServer = new PigServer(initString);
 		
 		File f1 = createFile(new String[]{"a:1","b:1","a:1"});
@@ -107,7 +109,7 @@
 	}
 	
 	@Test
-	public void testDriverMethod() throws Exception{
+	public void testDriverMethod() throws Throwable {
 		PigServer pigServer = new PigServer(initString);
 		File f = File.createTempFile("tmp", "");
 		PrintWriter pw = new PrintWriter(f);
@@ -130,7 +132,7 @@
 	
 	
 	@Test
-	public void testMapLookup() throws IOException{
+	public void testMapLookup() throws Throwable {
 		PigServer pigServer = new PigServer(initString);
 		DataBag b = BagFactory.getInstance().newDefaultBag();
 		DataMap colors = new DataMap();
@@ -211,12 +213,9 @@
 			return sb.toString();
 		}
 	}
-
-	
-	
-	
+
 	@Test
-	public void testBagFunctionWithFlattening() throws Exception{
+	public void testBagFunctionWithFlattening() throws Throwable {
 		PigServer pigServer = new PigServer(initString);
 		File queryLogFile = createFile(
 					new String[]{ 
@@ -259,20 +258,17 @@
 	    }
 	}
 	
-
-	
 	@Test
-	public void testSort() throws Exception{
+	public void testSort() throws Throwable {
 		testSortDistinct(false);
 	}
 	
-
 	@Test
-	public void testDistinct() throws Exception{
+	public void testDistinct() throws Throwable {
 		testSortDistinct(true);
 	}
-
-	private void testSortDistinct(boolean eliminateDuplicates) throws Exception{
+
+	private void testSortDistinct(boolean eliminateDuplicates) throws Throwable {
 		int LOOP_SIZE = 1024*16;
 		File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java Thu Jan 31 19:11:28 2008
@@ -37,7 +37,7 @@
     private String initString = "mapreduce";
     
     @Test
-    public void testNumericEq() throws Exception {
+    public void testNumericEq() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -68,7 +68,7 @@
     }
 
     @Test
-    public void testNumericNeq() throws Exception {
+    public void testNumericNeq() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -95,7 +95,7 @@
     }
 
     @Test
-    public void testNumericGt() throws Exception {
+    public void testNumericGt() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -123,7 +123,7 @@
     }
 
     @Test
-    public void testBinCond() throws Exception {
+    public void testBinCond() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -152,7 +152,7 @@
     
     
     @Test
-    public void testNestedBinCond() throws Exception {
+    public void testNestedBinCond() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -180,7 +180,7 @@
     
     
     @Test 
-    public void testNumericLt() throws Exception {
+    public void testNumericLt() throws Throwable {
     	PigServer pig = new PigServer(initString);
     	File tmpFile = File.createTempFile("test", "txt");
     	PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -210,7 +210,7 @@
 
     
     @Test
-    public void testNumericGte() throws Exception {
+    public void testNumericGte() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -240,7 +240,7 @@
     }
 
     @Test
-    public void testNumericLte() throws Exception {
+    public void testNumericLte() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java Thu Jan 31 19:11:28 2008
@@ -36,7 +36,7 @@
     private String initString = "mapreduce";
     
     @Test
-    public void testStringEq() throws Exception {
+    public void testStringEq() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -64,7 +64,7 @@
     }
     
     @Test
-    public void testStringNeq() throws Exception {
+    public void testStringNeq() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -92,7 +92,7 @@
     }
 
     @Test
-    public void testStringGt() throws Exception {
+    public void testStringGt() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -122,7 +122,7 @@
     
 
     @Test
-    public void testStringGte() throws Exception {
+    public void testStringGte() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -153,7 +153,7 @@
     }
 
     @Test
-    public void testStringLt() throws Exception {
+    public void testStringLt() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -182,7 +182,7 @@
     }
 
     @Test
-    public void testStringLte() throws Exception {
+    public void testStringLte() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Added: incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=617338&view=auto
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java (added)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java Thu Jan 31 19:11:28 2008
@@ -0,0 +1,31 @@
+package org.apache.pig.test;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.grunt.Grunt;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+
+public class TestGrunt extends TestCase {
+
+	
+	@Test 
+	public void testCopyFromLocal() throws Throwable {
+        PigServer server = new PigServer("MAPREDUCE");
+        PigContext context = server.getPigContext();
+        
+        String strCmd = "copyFromLocal /tmp/TestMe;";
+        
+        ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+        InputStreamReader reader = new InputStreamReader(cmd);
+        
+        Grunt grunt = new Grunt(new BufferedReader(reader), context);
+	
+        grunt.exec();
+	}
+}

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java Thu Jan 31 19:11:28 2008
@@ -36,7 +36,7 @@
     private String initString = "mapreduce";
     
     @Test
-    public void testAdd() throws Exception {
+    public void testAdd() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -58,7 +58,7 @@
     }
  
     @Test
-    public void testSubtract() throws Exception {
+    public void testSubtract() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -79,7 +79,7 @@
     }
  
     @Test
-    public void testMultiply() throws Exception {
+    public void testMultiply() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -101,7 +101,7 @@
     }
     
     @Test
-    public void testDivide() throws Exception {
+    public void testDivide() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java Thu Jan 31 19:11:28 2008
@@ -30,10 +30,13 @@
 import junit.framework.TestCase;
 
 import java.io.File;
+import java.io.IOException;
 import java.io.FileOutputStream;
 import java.util.Iterator;
 import java.util.Random;
 
+import org.apache.pig.backend.executionengine.ExecException;
+
 import org.apache.hadoop.conf.Configuration;
 //This class tests pig behavior with large file spanning multiple blocks along with group and count functions
 //Order and Distinct functions are also tested.
@@ -84,7 +87,12 @@
         
         dat.close();
     
-        pig = new PigServer(initString);
+        try {
+            pig = new PigServer(initString);
+        }
+        catch (ExecException e) {
+            throw new IOException("Failed to create Pig server", e);
+        }
 		fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
 		tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
 

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Thu Jan 31 19:11:28 2008
@@ -34,12 +34,13 @@
 import org.apache.pig.PigServer.ExecType;
 import org.apache.pig.impl.builtin.ShellBagEvalFunc;
 import org.apache.pig.impl.io.BufferedPositionedInputStream;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
 import org.apache.pig.impl.logicalLayer.LOCogroup;
 import org.apache.pig.impl.logicalLayer.LOEval;
 import org.apache.pig.impl.logicalLayer.LogicalOperator;
 import org.apache.pig.impl.logicalLayer.LogicalPlan;
 import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
+
 
 public class TestLogicalPlanBuilder extends junit.framework.TestCase {
 
@@ -198,13 +199,15 @@
     public void testQuery18() {
         String query = "FOREACH (group (load 'a') ALL PARALLEL 16) generate group;";
         LogicalPlan lp = buildPlan(query);
-        LogicalOperator lo = lp.getRoot().getInputs().get(0);
+        Map<OperatorKey, LogicalOperator> logicalOpTable = lp.getOpTable();
+        OperatorKey logicalKey = lp.getRoot();
+        LogicalOperator lo = logicalOpTable.get(logicalOpTable.get(logicalKey).getInputs().get(0));
+        
         if (lo instanceof LOCogroup) {
             assertTrue(((LOCogroup) lo).getRequestedParallelism() == 16);
         } else {
             fail("Error: Unexpected Parse Tree output");
         }
-
     }
     
     
@@ -425,7 +428,7 @@
     }
 
     
-    public void testQueryFail44() {
+    public void testQueryFail44() throws Throwable {
         PigServer pig = null;
         try {
             pig = new PigServer("local");
@@ -532,15 +535,18 @@
         LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
 
         try {
-            LogicalPlan lp = builder.parse(query, queryResults);
-            if (lp.getRoot() instanceof LOEval){
+            LogicalPlan lp = builder.parse("Test-Plan-Builder",
+                                           query,
+                                           aliases,
+                                           logicalOpTable);
+            if (logicalOpTable.get(lp.getRoot()) instanceof LOEval){
             	System.out.println(query);
-            	System.out.println(((LOEval)lp.getRoot()).getSpec());
+            	System.out.println(((LOEval)logicalOpTable.get(lp.getRoot())).getSpec());
             }
             if (lp.getAlias()!=null){
-            	queryResults.put(lp.getAlias(), new IntermedResult(lp, pigContext));
+            	aliases.put(lp.getAlias(), lp);
             }
-            //System.out.println(lp.root().outputSchema());
+            
             assertTrue(lp != null);
             return lp;
         } catch (IOException e) {
@@ -553,6 +559,6 @@
         return null;
     }
     
-    Map<String, IntermedResult> queryResults = new HashMap<String, IntermedResult>();
-
+    Map<String, LogicalPlan> aliases = new HashMap<String, LogicalPlan>();
+    Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();    
 }

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java Thu Jan 31 19:11:28 2008
@@ -29,7 +29,6 @@
 
 import junit.framework.TestCase;
 
-import org.apache.hadoop.fs.Path;
 import org.junit.Test;
 
 import org.apache.pig.EvalFunc;
@@ -40,18 +39,18 @@
 import org.apache.pig.builtin.COUNT;
 import org.apache.pig.data.DataAtom;
 import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Datum;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.io.BufferedPositionedInputStream;
 import org.apache.pig.impl.PigContext;
+import org.apache.pig.backend.datastorage.ElementDescriptor;
 
 public class TestMapReduce extends TestCase {
 
 	private String initString = "mapreduce";
-	
+
 	@Test
-    public void testBigGroupAll() throws Exception {
+    public void testBigGroupAll() throws Throwable {
         int LOOP_COUNT = 4*1024;
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
@@ -119,7 +118,7 @@
 		}
     }
     @Test
-    public void testStoreFunction() throws IOException {
+    public void testStoreFunction() throws Throwable {
     	PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", ".txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -146,7 +145,7 @@
         pig.deleteFile("frog");
     }
     @Test
-    public void testQualifiedFuncions() throws IOException {
+    public void testQualifiedFuncions() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", ".txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -171,7 +170,7 @@
     }
     
     @Test
-    public void testDefinedFunctions() throws IOException {
+    public void testDefinedFunctions() throws Throwable {
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", ".txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -195,9 +194,9 @@
         }
         assertEquals(count, MyStorage.COUNT);
     }
-    
+
     @Test
-    public void testPigServer() throws IOException {
+    public void testPigServer() throws Throwable {
         System.out.println("creating pig server");
         PigContext pigContext = new PigContext(ExecType.MAPREDUCE);
         PigServer pig = new PigServer(pigContext);
@@ -206,24 +205,12 @@
         assertTrue(capacity > 0);
         String sampleFileName = "/tmp/fileTest";
         if (!pig.existsFile(sampleFileName)) {
-            OutputStream os = pigContext.getDfs().create(new Path(sampleFileName));
+            ElementDescriptor path = pigContext.getDfs().asElement(sampleFileName);
+            OutputStream os = path.create();
             os.write("Ben was here!".getBytes());
             os.close();
         }
         long length = pig.fileSize(sampleFileName);
         assertTrue(length > 0);
     }
-    
-    @Test
-    public void testCreateNewRelation() throws IOException {
-        System.out.println("creating pig server");
-        PigServer pig = new PigServer(initString);
-		pig.deleteFile("/tmp/test_createNewRelation");
-        System.out.println("testing create new relation");
-        pig.newRelation("new_rel");
-        pig.insertTuple("new_rel", new Tuple("hello"));
-        pig.store("new_rel", "/tmp/test_createNewRelation");
-        assertTrue(pig.existsFile("/tmp/test_createNewRelation"));
-    }
-    
 }

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java Thu Jan 31 19:11:28 2008
@@ -38,7 +38,7 @@
     private PigServer pig;
     private File tmpFile;
 
-    public TestOrderBy() throws Exception {
+    public TestOrderBy() throws Throwable {
         DecimalFormat myFormatter = new DecimalFormat("0000000");
         for (int i = 0; i < DATALEN; i++) {
             DATA[0][i] = myFormatter.format(i);

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPi.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPi.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPi.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPi.java Thu Jan 31 19:11:28 2008
@@ -24,12 +24,15 @@
 
 import org.apache.pig.impl.io.FileLocalizer;
 
+import org.apache.pig.backend.executionengine.ExecException;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import junit.framework.TestCase;
 
 import java.io.File;
+import java.io.IOException;
 import java.io.FileOutputStream;
 import java.util.Iterator;
 import java.util.Random;
@@ -90,8 +93,14 @@
         
         dat.close();
         
-        pig = new PigServer(initString);
-		fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
+        try {
+            pig = new PigServer(initString);
+        }
+        catch (ExecException e) {
+            throw new IOException("Failed to create Pig Server", e);
+        }
+        
+        fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
 		tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
 
         datFile.delete();

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java Thu Jan 31 19:11:28 2008
@@ -178,7 +178,7 @@
     }
 
 
-    public void testLocalStore() throws Exception{
+    public void testLocalStore() throws Throwable {
     	PigServer pig = new PigServer("local");
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Thu Jan 31 19:11:28 2008
@@ -18,8 +18,15 @@
 
 
 public class TestPigServer extends TestCase {
-    private PigServer pig = new PigServer();
-
+    private PigServer pig = null;
+    
+    private void initPigServer() throws Throwable {
+        if (pig == null) {
+            pig = new PigServer();
+        }
+    }
+    
+    
     private final static String FILE_SEPARATOR = System.getProperty("file.separator");
     
     // make sure that name is included or not (depending on flag "included") 
@@ -79,12 +86,13 @@
      * The jar file to register is not present
      */
     @Test
-    public void testRegisterJarFileNotPresent() throws Exception {
+    public void testRegisterJarFileNotPresent() throws Throwable {
         // resister a jar file that does not exist
         
         String jarName = "BadFileNameTestJarNotPresent.jar";
         
         // jar name is not present to start with
+        initPigServer();
         verifyStringContained(pig.getPigContext().extraJars, jarName, false);
 
         boolean exceptionRaised = false;
@@ -103,13 +111,15 @@
      * in this case name of jar file is relative to current working dir
      */
     @Test
-    public void testRegisterJarLocalDir() throws Exception {
+    public void testRegisterJarLocalDir() throws Throwable {
         String dir1 = "test1_register_jar_local";
         String dir2 = "test2_register_jar_local";
         String jarLocation = dir1 + FILE_SEPARATOR +
                               dir2 + FILE_SEPARATOR;
         String jarName = "TestRegisterJarLocal.jar";
         
+        initPigServer();
+        
         createFakeJarFile(jarLocation, jarName);
         
         verifyStringContained(pig.getPigContext().extraJars, jarName, false);
@@ -136,7 +146,7 @@
      * only one of them is registered.
      */
     @Test
-    public void testRegisterJarFromResources () throws Exception {
+    public void testRegisterJarFromResources () throws Throwable {
         String dir = "test_register_jar_res_dir";
         String subDir1 = "test_register_jar_res_sub_dir1";
         String subDir2 = "test_register_jar_res_sub_dir2";
@@ -144,6 +154,8 @@
         String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
         String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
         
+        initPigServer();
+        
         createFakeJarFile(jarLocation1, jarName);
         createFakeJarFile(jarLocation2, jarName);
         
@@ -176,13 +188,15 @@
      * @throws Exception
      */
     @Test
-    public void testRegisterJarResourceInJar() throws Exception {
+    public void testRegisterJarResourceInJar() throws Throwable {
         String dir = "test_register_jar_res_in_jar";
         String subDir = "sub_dir";
         String jarName = "TestRegisterJarNonEmpty.jar";
         String className = "TestRegisterJar";
         String javaSrc = "package " + subDir + "; class " + className + " { }";
 
+        initPigServer();
+        
         // create dirs
         (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
 

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java Thu Jan 31 19:11:28 2008
@@ -20,6 +20,7 @@
 
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.IOException;
 import java.io.FileWriter;
 import java.io.PrintWriter;
 import java.util.Iterator;
@@ -28,6 +29,7 @@
 
 import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.backend.executionengine.ExecException;
 
 import junit.framework.TestCase;
 
@@ -37,9 +39,14 @@
 	@Override
 	protected void setUp() throws Exception {
 		super.setUp();
-		pig = new PigServer();
+		
+		try {
+		    pig = new PigServer();
+		}
+		catch (ExecException e) {
+		    throw new IOException("Failed to create Pig Server", e);
+		}
 	}
-	
 	@Test
 	public void testLongEvalSpec() throws Exception{
 		File f = File.createTempFile("tmp", "");

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestStore.java Thu Jan 31 19:11:28 2008
@@ -18,12 +18,14 @@
 package org.apache.pig.test;
 
 import java.io.File;
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Iterator;
 
 import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.backend.executionengine.ExecException;
 
 import junit.framework.TestCase;
 
@@ -35,7 +37,7 @@
 	String fileName;
 	String tmpFile1, tmpFile2;
 	PigServer pig;
-	
+
 	public void testSingleStore() throws Exception{
 		pig.registerQuery("A = load " + fileName + ";");
 		
@@ -103,8 +105,14 @@
 		for (int i=0;i<LOOP_COUNT; i++){
 			pw.println(i + "\t" + i);
 		}
-		pw.close();
-		pig = new PigServer(initString);
+		pw.close();
+		try {
+		    pig = new PigServer(initString);
+		}
+		catch (ExecException e) {
+		    throw new IOException("Failed to create Pig Server", e);
+		}
+		
 		fileName = "'" + FileLocalizer.hadoopify(f.toString(), pig.getPigContext()) + "'";
 		tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
 		tmpFile2 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";