You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/02/01 04:11:39 UTC
svn commit: r617338 [5/5] - in /incubator/pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/ src/org/apache/pig/backend/datastorage/
src/org/apache/pig/backend/executionengine/
src/org/apache/pig/backend/hadoop/ src/org/apache/pig/backend/ha...
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java Thu Jan 31 19:11:28 2008
@@ -18,6 +18,7 @@
package org.apache.pig.impl.logicalLayer;
import java.util.List;
+import java.util.Map;
import org.apache.pig.impl.logicalLayer.schema.TupleSchema;
@@ -26,25 +27,30 @@
public class LOUnion extends LogicalOperator {
private static final long serialVersionUID = 1L;
- public LOUnion(List<LogicalOperator> inputsIn) {
- super(inputsIn);
+ public LOUnion(Map<OperatorKey, LogicalOperator> opTable,
+ String scope,
+ long id,
+ List<OperatorKey> inputsIn) {
+ super(opTable, scope, id, inputsIn);
}
@Override
public String name() {
- return "Union";
+ return "Union " + scope + "-" + id;
}
@Override
public TupleSchema outputSchema() {
if (schema == null) {
- TupleSchema longest = getInputs().get(0).outputSchema();
+ TupleSchema longest = opTable.get(getInputs().get(0)).outputSchema();
int current = 0;
- for (LogicalOperator lo:getInputs()) {
- if (lo != null && lo.outputSchema() != null
- && lo.outputSchema().numFields() > current) {
- longest = lo.outputSchema();
- current = longest.numFields();
+ for (OperatorKey opKey: getInputs()) {
+ LogicalOperator lo = opTable.get(opKey);
+
+ if (lo != null && lo.outputSchema() != null &&
+ lo.outputSchema().numFields() > current) {
+ longest = lo.outputSchema();
+ current = longest.numFields();
}
}
schema = longest.copy();
@@ -58,7 +64,7 @@
public int getOutputType() {
int outputType = FIXED;
for (int i = 0; i < getInputs().size(); i++) {
- switch (getInputs().get(i).getOutputType()) {
+ switch (opTable.get(getInputs().get(i)).getOutputType()) {
case FIXED:
continue;
case MONOTONE:
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOVisitor.java Thu Jan 31 19:11:28 2008
@@ -20,7 +20,6 @@
import java.util.List;
import java.util.Iterator;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
/**
* A visitor mechanism for navigating and operating on a tree of Logical
@@ -69,17 +68,6 @@
basicVisit(u);
}
- /**
- * Only LORead.visit() and subclass implementations of this function
- * should ever call this method.
- */
- public void visitRead(LORead r) {
- // I'm assuming LORead is always at the head of a list of LO operators.
- IntermedResult ir = r.getReadFrom();
- if (ir != null) {
- ir.lp.getRoot().visit(this);
- }
- }
/**
* Only LOLoad.visit() and subclass implementations of this function
@@ -104,6 +92,11 @@
public void visitSplit(LOSplit s) {
basicVisit(s);
}
+
+ public void visitSplitOutput(LOSplitOutput s) {
+ basicVisit(s);
+ }
+
/**
* Only LOStore.visit() and subclass implementations of this function
@@ -114,10 +107,11 @@
}
private void basicVisit(LogicalOperator lo) {
- List<LogicalOperator> inputs = lo.getInputs();
- Iterator<LogicalOperator> i = inputs.iterator();
+ List<OperatorKey> inputs = lo.getInputs();
+ Iterator<OperatorKey> i = inputs.iterator();
+
while (i.hasNext()) {
- LogicalOperator input = i.next();
+ LogicalOperator input = lo.getOpTable().get(i.next());
input.visit(this);
}
}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Thu Jan 31 19:11:28 2008
@@ -21,12 +21,14 @@
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import org.apache.pig.impl.logicalLayer.schema.TupleSchema;
+import org.apache.pig.backend.executionengine.ExecScopedLogicalOperator;
-abstract public class LogicalOperator implements Serializable {
+abstract public class LogicalOperator implements Serializable, ExecScopedLogicalOperator {
public String alias = null;
public static final int FIXED = 1;
@@ -36,19 +38,64 @@
protected int requestedParallelism = -1;
protected TupleSchema schema = null;
- protected List<LogicalOperator> inputs;
+ protected List<OperatorKey> inputs;
- protected LogicalOperator() {
- this.inputs = new ArrayList<LogicalOperator> ();
- } protected LogicalOperator(List<LogicalOperator> inputs) {
+ protected Map<OperatorKey, LogicalOperator> opTable;
+
+ protected String scope;
+ protected long id;
+
+ protected LogicalOperator(Map<OperatorKey, LogicalOperator> opTable,
+ String scope,
+ long id) {
+ this.inputs = new ArrayList<OperatorKey> ();
+ this.opTable = opTable;
+ this.scope = scope;
+ this.id = id;
+
+ opTable.put(getOperatorKey(), this);
+ }
+
+ protected LogicalOperator(Map<OperatorKey, LogicalOperator> opTable,
+ String scope,
+ long id,
+ List<OperatorKey> inputs) {
+ this.opTable = opTable;
this.inputs = inputs;
+ this.scope = scope;
+ this.id = id;
+
+ opTable.put(getOperatorKey(), this);
}
- protected LogicalOperator(LogicalOperator input) {
- this.inputs = new ArrayList<LogicalOperator> ();
+ protected LogicalOperator(Map<OperatorKey, LogicalOperator> opTable,
+ String scope,
+ long id,
+ OperatorKey input) {
+ this.opTable = opTable;
+ this.inputs = new ArrayList<OperatorKey> ();
inputs.add(input);
- }
+ this.scope = scope;
+ this.id = id;
+ opTable.put(getOperatorKey(), this);
+ }
+
+ @Override
+ public OperatorKey getOperatorKey() {
+ return new OperatorKey(scope, id);
+ }
+
+ @Override
+ public String getScope() {
+ return this.scope;
+ }
+
+ @Override
+ public long getId() {
+ return this.id;
+ }
+
public String getAlias() {
return alias;
}
@@ -78,13 +125,17 @@
public abstract TupleSchema outputSchema();
public String name() {
- return "ROOT";
+ return "ROOT " + scope + "-" + id;
}
- public List<LogicalOperator> getInputs() {
+ public List<OperatorKey> getInputs() {
return inputs;
}
+ public Map<OperatorKey, LogicalOperator> getOpTable() {
+ return opTable;
+ }
+
public String arguments() {
return "";
}
@@ -92,7 +143,7 @@
public List<String> getFuncs() {
List<String> funcs = new LinkedList<String>();
for (int i = 0; i < inputs.size(); i++) {
- funcs.addAll(inputs.get(i).getFuncs());
+ funcs.addAll(opTable.get(inputs.get(i)).getFuncs());
}
return funcs;
}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Thu Jan 31 19:11:28 2008
@@ -20,34 +20,44 @@
import java.io.Serializable;
import java.util.LinkedList;
import java.util.List;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Map;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.backend.executionengine.ExecLogicalPlan;
-
-
-public class LogicalPlan implements Serializable{
+public class LogicalPlan implements Serializable, ExecLogicalPlan{
private static final long serialVersionUID = 1L;
- protected LogicalOperator root;
+ protected OperatorKey root;
+ protected Map<OperatorKey, LogicalOperator> opTable;
protected PigContext pigContext = null;
protected String alias = null;
-
- public LogicalOperator getRoot() {
+
+ public OperatorKey getRoot() {
return root;
}
- public LogicalPlan(LogicalOperator rootIn, PigContext pigContext) {
+ public LogicalPlan(OperatorKey rootIn,
+ Map<OperatorKey, LogicalOperator> opTable ,
+ PigContext pigContext) {
this.pigContext = pigContext;
- root = rootIn;
- alias = root.alias;
+ this.root = rootIn;
+ this.opTable = opTable;
+ alias = opTable.get(root).alias;
+ }
+
+ public Map<OperatorKey, LogicalOperator> getOpTable() {
+ return this.opTable;
}
- public void setRoot(LogicalOperator newRoot) {
+ public void setRoot(OperatorKey newRoot) {
root = newRoot;
}
-
+
public PigContext getPigContext() {
return pigContext;
}
@@ -60,35 +70,41 @@
alias = newAlias;
}
-
public List<String> getFuncs() {
if (root == null) return new LinkedList<String>();
- else return root.getFuncs();
+ else return opTable.get(root).getFuncs();
}
// indentation for root is 0
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
- sb.append(root.name() +"(" + root.arguments() +")\n");
+ sb.append(opTable.get(root).name() +"(" + opTable.get(root).arguments() +")\n");
sb.append(appendChildren(root, 1));
return sb.toString();
}
- public String appendChildren(LogicalOperator parent, int indentation) {
+ public String appendChildren(OperatorKey parent, int indentation) {
StringBuffer sb = new StringBuffer();
- List<LogicalOperator> children = parent.getInputs();
+ List<OperatorKey> children = opTable.get(parent).getInputs();
for(int i=0; i != children.size(); i++) {
for(int j=0; j != indentation; j++) {
sb.append("\t");
}
- sb.append(children.get(i).name() + "(" + children.get(i).arguments()+ ")\n");
+
+ sb.append(opTable.get(children.get(i)).name() +
+ "(" + opTable.get(children.get(i)).arguments()+ ")\n");
sb.append(appendChildren(children.get(i), indentation+1));
}
return sb.toString();
}
public int getOutputType(){
- return root.getOutputType();
+ return opTable.get(root).getOutputType();
}
+ public void explain(OutputStream out) {
+ LOVisitor lprinter = new LOPrinter(new PrintStream(out));
+
+ opTable.get(root).visit(lprinter);
+ }
}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java Thu Jan 31 19:11:28 2008
@@ -24,7 +24,6 @@
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.parser.QueryParser;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
/**
@@ -38,16 +37,13 @@
this.pigContext = pigContext;
}
- public LogicalPlan parse(String query, Map<String, IntermedResult> aliases)
+ public LogicalPlan parse(String scope,
+ String query,
+ Map<String, LogicalPlan> aliases,
+ Map<OperatorKey, LogicalOperator> opTable)
throws IOException, ParseException {
- /*
- if(query.matches(".*select.*")) {
- query = sqlToPigLatin(query);
- //System.err.println(query);
- }
- */
ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());
- QueryParser parser = new QueryParser(in, pigContext, aliases);
+ QueryParser parser = new QueryParser(in, pigContext, scope, aliases, opTable);
return parser.Parse();
}
}
Added: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java?rev=617338&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/OperatorKey.java Thu Jan 31 19:11:28 2008
@@ -0,0 +1,49 @@
+package org.apache.pig.impl.logicalLayer;
+
+import java.io.Serializable;
+
+public class OperatorKey implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public String scope;
+ public long id;
+
+ public OperatorKey() {
+ this("", -1);
+ }
+
+ public OperatorKey(String scope, long id) {
+ this.scope = scope;
+ this.id = id;
+ }
+
+ @Override
+ public String toString() {
+ return scope + "-" + id;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (! (obj instanceof OperatorKey)) {
+ return false;
+ }
+ else {
+ OperatorKey otherKey = (OperatorKey) obj;
+ return this.scope.equals(otherKey.scope) &&
+ this.id == otherKey.id;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return this.toString().hashCode();
+ }
+
+ public String getScope() {
+ return scope;
+ }
+
+ public long getId() {
+ return id;
+ }
+}
\ No newline at end of file
Added: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java?rev=617338&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/NodeIdGenerator.java Thu Jan 31 19:11:28 2008
@@ -0,0 +1,32 @@
+package org.apache.pig.impl.logicalLayer.parser;
+
+import java.util.Map;
+import java.util.HashMap;
+
+public class NodeIdGenerator {
+
+ private Map<String, Long> scopeToIdMap;
+ private static NodeIdGenerator theGenerator = new NodeIdGenerator();
+
+ private NodeIdGenerator() {
+ scopeToIdMap = new HashMap<String, Long>();
+ }
+
+ public static NodeIdGenerator getGenerator() {
+ return theGenerator;
+ }
+
+ public long getNextNodeId(String scope) {
+ Long val = scopeToIdMap.get(scope);
+
+ long nextId = 0;
+
+ if (val != null) {
+ nextId = val.longValue();
+ }
+
+ scopeToIdMap.put(scope, nextId + 1);
+
+ return nextId;
+ }
+}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Thu Jan 31 19:11:28 2008
@@ -40,19 +40,33 @@
import org.apache.pig.data.*;
import org.apache.pig.impl.PigContext;
import org.apache.pig.PigServer.ExecType;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
import org.apache.pig.impl.io.*;
import org.apache.pig.builtin.*;
import org.apache.pig.impl.builtin.*;
+import org.apache.pig.impl.logicalLayer.LogicalPlan;
public class QueryParser {
private PigContext pigContext;
- private Map<String, IntermedResult> aliases;
+ private Map<String, LogicalPlan> aliases;
+ private Map<OperatorKey, LogicalOperator> opTable;
+ private String scope;
+ private NodeIdGenerator nodeIdGen;
+
+ private long getNextId() {
+ return nodeIdGen.getNextNodeId(scope);
+ }
- public QueryParser(InputStream in, PigContext pigContext, Map aliases) {
+ public QueryParser(InputStream in,
+ PigContext pigContext,
+ String scope,
+ Map<String, LogicalPlan> aliases,
+ Map<OperatorKey, LogicalOperator> opTable) {
this(in);
this.pigContext = pigContext;
this.aliases = aliases;
+ this.opTable = opTable;
+ this.scope = scope;
+ this.nodeIdGen = NodeIdGenerator.getGenerator();
}
public class EvalSpecAndSchema{
@@ -61,10 +75,43 @@
}
public class CogroupInput {
- public LogicalOperator op;
+ public OperatorKey op;
public EvalSpec spec;
}
+ private static String removeQuotes(String str) {
+ if (str.startsWith("\'") && str.endsWith("\'"))
+ return str.substring(1, str.length() - 1);
+ else
+ return str;
+ }
+
+ public static LogicalPlan generateStorePlan(Map<OperatorKey, LogicalOperator> opTable,
+ String scope,
+ LogicalPlan readFrom,
+ String fileName,
+ String func,
+ PigContext pigContext) throws IOException {
+ if (func == null) {
+ func = PigStorage.class.getName();
+ }
+
+ fileName = removeQuotes(fileName);
+
+ long storeNodeId = NodeIdGenerator.getGenerator().getNextNodeId(scope);
+
+ LogicalOperator root = new LOStore(opTable,
+ scope,
+ storeNodeId,
+ readFrom.getRoot(),
+ new FileSpec(fileName, func),
+ false);
+
+ LogicalPlan storePlan = new LogicalPlan(root.getOperatorKey(), opTable, pigContext);
+
+ return storePlan;
+ }
+
static String unquote(String s) {
return s.substring(1, s.length()-1);
}
@@ -74,20 +121,11 @@
}
- LogicalOperator makeLORead(String alias) throws ParseException {
- if (!aliases.containsKey(alias)) throw new ParseException("Unrecognized alias: " + alias);
-
- LORead readOp = new LORead(aliases.get(alias));
- readOp.setAlias(alias);
- return readOp;
- }
-
-
String massageFilename(String filename, PigContext pigContext) throws IOException, ParseException {
if (pigContext.getExecType() != ExecType.LOCAL) {
if (filename.startsWith(FileLocalizer.LOCAL_PREFIX)) {
- filename = FileLocalizer.hadoopify(filename, pigContext);
+ filename = FileLocalizer.hadoopify(filename, pigContext);
}
else
{
@@ -104,7 +142,7 @@
LogicalOperator parseCogroup(ArrayList<CogroupInput> gis) throws ParseException{
int n = gis.size();
- List<LogicalOperator> los = new ArrayList<LogicalOperator>();
+ List<OperatorKey> los = new ArrayList<OperatorKey>();
ArrayList<EvalSpec> specs = new ArrayList<EvalSpec>();
for (int i = 0; i < n ; i++){
@@ -114,11 +152,11 @@
specs.add(gi.spec);
}
- return new LOCogroup(los, specs);
+ return new LOCogroup(opTable, scope, getNextId(), los, specs);
}
- LogicalOperator rewriteCross(ArrayList<LogicalOperator> inputs) throws IOException, ParseException{
+ LogicalOperator rewriteCross(ArrayList<OperatorKey> inputs) throws IOException, ParseException{
ArrayList<CogroupInput> gis = new ArrayList<CogroupInput>();
int n = inputs.size();
@@ -140,23 +178,23 @@
return rewriteJoin(gis);
}
- LogicalOperator rewriteDistinct(LogicalOperator input){
+ LogicalOperator rewriteDistinct(OperatorKey input){
//First group the input on *
- ArrayList<LogicalOperator> inputs = new ArrayList<LogicalOperator>();
+ ArrayList<OperatorKey> inputs = new ArrayList<OperatorKey>();
inputs.add(input);
ArrayList<EvalSpec> groupSpecs = new ArrayList<EvalSpec>();
groupSpecs.add(new GenerateSpec(new StarSpec()).getGroupBySpec());
- LogicalOperator groupedInput = new LOCogroup(inputs, groupSpecs);
+ LogicalOperator groupedInput = new LOCogroup(opTable, scope, getNextId(), inputs, groupSpecs);
//then generate the flattened group
EvalSpec projectSpec = new ProjectSpec(0);
projectSpec.setFlatten(true);
- return new LOEval(groupedInput, new GenerateSpec(projectSpec));
+ return new LOEval(opTable, scope, getNextId(), groupedInput.getOperatorKey() , new GenerateSpec(projectSpec));
}
@@ -172,7 +210,7 @@
flattenedColumns.add(column);
}
- return new LOEval(parseCogroup(gis),new GenerateSpec(flattenedColumns));
+ return new LOEval(opTable, scope, getNextId(), parseCogroup(gis).getOperatorKey(),new GenerateSpec(flattenedColumns));
}
void assertAtomic(EvalSpec spec, boolean desiredAtomic) throws ParseException{
@@ -199,10 +237,10 @@
return spec.addSpec(successor);
}
- void addSplitOutput(IntermedResult intermedResult, LOSplit splitOp, String alias, Cond cond, int index){
+ void addSplitOutput(LogicalPlan lp, LOSplit splitOp, String alias, Cond cond, int index){
splitOp.addCond(cond);
- LORead readOp = new LORead(intermedResult, index);
- aliases.put(alias, new IntermedResult(new LogicalPlan(readOp, pigContext), pigContext));
+ LOSplitOutput splitOut = new LOSplitOutput(opTable, scope, getNextId(), lp.getRoot(), index);
+ aliases.put(alias, new LogicalPlan(splitOut.getOperatorKey(), opTable, pigContext));
}
}
@@ -290,27 +328,27 @@
| (root = Expr() ";")
| (<SPLIT> root = SplitClause() ";")
)
- { return new LogicalPlan(root, pigContext); }
+ { return new LogicalPlan(root.getOperatorKey(), opTable, pigContext); }
}
LogicalOperator SplitClause():
-{LogicalOperator input; Cond cond; Token alias; LOSplit splitOp; IntermedResult intermedResult; int i=0;}
+{LogicalOperator input; Cond cond; Token alias; LOSplit splitOp; LogicalPlan lp; int i=0;}
{
(
input = NestedExpr() <INTO>
{
- splitOp = new LOSplit(input);
- intermedResult = new IntermedResult(new LogicalPlan(splitOp, pigContext), pigContext);
+ splitOp = new LOSplit(opTable, scope, getNextId(), input.getOperatorKey());
+ lp = new LogicalPlan(splitOp.getOperatorKey(), opTable, pigContext);
}
alias = <IDENTIFIER> <IF> cond = PCond(input.outputSchema(),null)
{
- addSplitOutput(intermedResult, splitOp, alias.image, cond, i);
+ addSplitOutput(lp, splitOp, alias.image, cond, i);
i++;
}
(
"," alias = <IDENTIFIER> <IF> cond = PCond(input.outputSchema(),null)
{
- addSplitOutput(intermedResult, splitOp, alias.image, cond, i);
+ addSplitOutput(lp, splitOp, alias.image, cond, i);
i++;
}
)+
@@ -343,8 +381,16 @@
{
t1 = <IDENTIFIER>
{
- op = makeLORead(t1.image);
- return op;
+ LogicalOperator aliasOp;
+ String alias = t1.image;
+
+ aliasOp = opTable.get(aliases.get(alias).getRoot());
+
+ if (aliasOp == null) {
+ throw new ParseException("Unrecognized alias " + alias);
+ }
+
+ return aliasOp;
}
}
@@ -360,7 +406,7 @@
| ((<GROUP> | <COGROUP>) op = CogroupClause())
| (<FILTER> op = FilterClause())
| (<ORDER> op = OrderClause())
-| (<DISTINCT> op = NestedExpr() {op = rewriteDistinct(op);})
+| (<DISTINCT> op = NestedExpr() {op = rewriteDistinct(op.getOperatorKey());})
| (<CROSS> op = CrossClause())
| (<JOIN> op = JoinClause())
| (<UNION> op = UnionClause())
@@ -389,7 +435,7 @@
funcSpec += continuous ? "('\t','\n','0')" : "()";
}
- lo = new LOLoad(new FileSpec(massageFilename(filename, pigContext), funcSpec));
+ lo = new LOLoad(opTable, scope, getNextId(), new FileSpec(massageFilename(filename, pigContext), funcSpec));
if (continuous)
lo.setOutputType(LogicalOperator.MONOTONE);
return lo;
@@ -420,7 +466,7 @@
input = NestedExpr()
<BY> cond = PCond(input.outputSchema(),null)
{
- return new LOEval(input, new FilterSpec(cond));
+ return new LOEval(opTable, scope, getNextId(), input.getOperatorKey(), new FilterSpec(cond));
}
}
@@ -491,27 +537,34 @@
}
-CogroupInput GroupItem() : {LogicalOperator op; GenerateSpec gs; EvalSpec es; CogroupInput cogroupInput = new CogroupInput();}
+CogroupInput GroupItem() : {LogicalOperator op; GenerateSpec gs; EvalSpec es; LogicalOperator cgOp; EvalSpec cgSpec;}
{
(
- cogroupInput.op = NestedExpr()
+ cgOp = NestedExpr()
(
( <BY>
(
- LOOKAHEAD ( "(" FlattenedGenerateItemList(cogroupInput.op.outputSchema()) ")" )
- ( "(" gs = FlattenedGenerateItemList(cogroupInput.op.outputSchema(), null) ")" )
-| (es = FlattenedGenerateItem(cogroupInput.op.outputSchema(), null) {gs = new GenerateSpec(es);})
+ LOOKAHEAD ( "(" FlattenedGenerateItemList(cgOp.outputSchema()) ")" )
+ ( "(" gs = FlattenedGenerateItemList(cgOp.outputSchema(), null) ")" )
+| (es = FlattenedGenerateItem(cgOp.outputSchema(), null) {gs = new GenerateSpec(es);})
)
)
| <ALL> {gs = new GenerateSpec(new ConstSpec("all"));}
| <ANY> {gs = new GenerateSpec(new FuncEvalSpec(pigContext, GFAny.class.getName(), null));}
)
{
- cogroupInput.spec = gs.getGroupBySpec();
+ cgSpec = gs.getGroupBySpec();
}
- [<INNER> {cogroupInput.spec.setInner(true);} | <OUTER>]
+ [<INNER> {cgSpec.setInner(true);} | <OUTER>]
)
- {return cogroupInput;}
+ {
+ CogroupInput cogroupInput = new CogroupInput();
+
+ cogroupInput.spec = cgSpec;
+ cogroupInput.op = cgOp.getOperatorKey();
+
+ return cogroupInput;
+ }
}
@@ -543,16 +596,16 @@
)
{
- return new LOSort(op, sortSpec);
+ return new LOSort(opTable, scope, getNextId(), op.getOperatorKey(), sortSpec);
}
}
-LogicalOperator CrossClause() : {LogicalOperator op; ArrayList<LogicalOperator> inputs = new ArrayList<LogicalOperator>();}
+LogicalOperator CrossClause() : {LogicalOperator op; ArrayList<OperatorKey> inputs = new ArrayList<OperatorKey>();}
{
(
- op = NestedExpr() { inputs.add(op); }
- ("," op = NestedExpr() { inputs.add(op); })*
+ op = NestedExpr() { inputs.add(op.getOperatorKey()); }
+ ("," op = NestedExpr() { inputs.add(op.getOperatorKey()); })*
)
{return rewriteCross(inputs);}
}
@@ -564,11 +617,11 @@
{return rewriteJoin(gis);}
}
-LogicalOperator UnionClause() : {LogicalOperator op; ArrayList<LogicalOperator> inputs = new ArrayList<LogicalOperator>();}
+LogicalOperator UnionClause() : {LogicalOperator op; ArrayList<OperatorKey> inputs = new ArrayList<OperatorKey>();}
{
- (op = NestedExpr() { inputs.add(op); }
- ("," op = NestedExpr() { inputs.add(op); })*)
- {return new LOUnion(inputs);}
+ (op = NestedExpr() { inputs.add(op.getOperatorKey()); }
+ ("," op = NestedExpr() { inputs.add(op.getOperatorKey()); })*)
+ {return new LOUnion(opTable, scope, getNextId(), inputs);}
}
@@ -579,7 +632,7 @@
spec = NestedBlock(input.outputSchema())
)
{
- op = new LOEval(input, spec);
+ op = new LOEval(opTable, scope, getNextId(), input.getOperatorKey(), spec);
return op;
}
}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java Thu Jan 31 19:11:28 2008
@@ -19,36 +19,55 @@
import java.io.IOException;
import java.io.Serializable;
+import java.util.Map;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
import org.apache.pig.impl.util.LineageTracer;
+import org.apache.pig.backend.executionengine.ExecPhysicalOperator;
+import org.apache.pig.backend.local.executionengine.POVisitor;
+public abstract class PhysicalOperator implements Serializable, ExecPhysicalOperator {
-abstract public class PhysicalOperator implements Serializable {
-
- public PhysicalOperator[] inputs = null;
- int outputType;
+ public Map<OperatorKey, ExecPhysicalOperator> opTable;
+ public OperatorKey[] inputs = null;
+ public int outputType;
protected LineageTracer lineageTracer = null;
+ protected String scope;
+ protected long id;
- /**
- *
- * This constructor should go away when we eventually implement CQ even for the mapreduce exec type
- */
- public PhysicalOperator(){
- outputType = LogicalOperator.FIXED;
+ public PhysicalOperator(String scope,
+ long id,
+ Map<OperatorKey, ExecPhysicalOperator> opTable,
+ int outputType){
+ this.scope = scope;
+ this.id = id;
+ this.opTable = opTable;
+ this.outputType = outputType;
+
+ opTable.put(getOperatorKey(), this);
}
-
- public PhysicalOperator(int outputType){
- this.outputType = outputType;
+ public OperatorKey getOperatorKey() {
+ return new OperatorKey(scope, id);
+ }
+
+ @Override
+ public String getScope() {
+ return this.scope;
+ }
+
+ @Override
+ public long getId() {
+ return this.id;
}
- public boolean open(boolean continueFromLast) throws IOException {
+ public boolean open() throws IOException {
// call open() on all inputs
if (inputs != null) {
for (int i = 0; i < inputs.length; i++) {
- if (!inputs[i].open(continueFromLast))
+ if (! ((PhysicalOperator)opTable.get(inputs[i])).open())
return false;
}
}
@@ -61,7 +80,7 @@
// call close() on all inputs
if (inputs != null)
for (int i = 0; i < inputs.length; i++)
- inputs[i].close();
+ ((PhysicalOperator)opTable.get(inputs[i])).close();
}
public void setLineageTracer(LineageTracer lineageTracer) {
@@ -72,5 +91,5 @@
return outputType;
}
- public abstract void visit(POVisitor v);
+ public abstract void visit(POVisitor v, String prfix);
}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java Thu Jan 31 19:11:28 2008
@@ -26,14 +26,7 @@
public abstract class PlanCompiler {
- PigContext pigContext;
- protected PlanCompiler(PigContext pigContext) {
- this.pigContext = pigContext;
- }
+
public abstract PhysicalOperator compile(LogicalOperator lo, Map queryResults) throws IOException;
- // various helper methods for map-reduce and hadoop compilation:
- public static String getTempFile(PigContext pigContext) throws IOException {
- return FileLocalizer.getTemporaryPath(null, pigContext).toString();
- }
}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Thu Jan 31 19:11:28 2008
@@ -38,8 +38,8 @@
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
+import org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigMapReduce;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.mapreduceExec.PigMapReduce;
public class JarManager {
Modified: incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java Thu Jan 31 19:11:28 2008
@@ -22,7 +22,7 @@
import org.apache.pig.PigServer;
import org.apache.pig.impl.PigContext;
import org.apache.pig.tools.grunt.GruntParser;
-
+import org.apache.pig.backend.executionengine.ExecException;
public class Grunt
@@ -31,7 +31,7 @@
PigServer pig;
GruntParser parser;
- public Grunt(BufferedReader in, PigContext pigContext)
+ public Grunt(BufferedReader in, PigContext pigContext) throws ExecException
{
this.in = in;
this.pig = new PigServer(pigContext);
Modified: incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Thu Jan 31 19:11:28 2008
@@ -4,20 +4,27 @@
import java.io.InputStream;
import java.io.Reader;
import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.RunningJob;
+
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigscript.parser.PigScriptParser;
import org.apache.pig.tools.pigscript.parser.PigScriptParserTokenManager;
+import org.apache.pig.backend.datastorage.DataStorage;
+import org.apache.pig.backend.datastorage.DataStorageException;
+import org.apache.pig.backend.datastorage.ElementDescriptor;
+import org.apache.pig.backend.datastorage.ContainerDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
public class GruntParser extends PigScriptParser {
@@ -72,9 +79,18 @@
public void setParams(PigServer pigServer)
{
mPigServer = pigServer;
+
mDfs = mPigServer.getPigContext().getDfs();
+ mLfs = mPigServer.getPigContext().getLfs();
mConf = mPigServer.getPigContext().getConf();
- mJobClient = mPigServer.getPigContext().getJobClient();
+
+ // TODO: this violates the abstraction layer decoupling between
+ // front end and back end and needs to be changed.
+ // Right now I am not clear on how the Job Id comes from to tell
+ // the back end to kill a given job (mJobClient is used only in
+ // processKill)
+ //
+ mJobClient = ((HExecutionEngine)(mPigServer.getPigContext().getExecutionEngine())).getJobClient();
}
public void prompt()
@@ -137,50 +153,74 @@
protected void processCat(String path) throws IOException
{
- byte buffer[] = new byte[65536];
- Path dfsPath = new Path(path);
- int rc;
-
- if (!mDfs.exists(dfsPath))
- throw new IOException("Directory " + path + " does not exist.");
-
- if (mDfs.getFileStatus(dfsPath).isDir())
- {
- FileStatus fileStat[] = mDfs.listStatus(dfsPath);
- for (int j = 0; j < fileStat.length; j++)
- {
- Path curPath = fileStat[j].getPath();
- if (!mDfs.isFile(curPath)) continue;
- FSDataInputStream is = mDfs.open(curPath);
- while ((rc = is.read(buffer)) > 0)
+ try {
+ byte buffer[] = new byte[65536];
+ ElementDescriptor dfsPath = mDfs.asElement(path);
+ int rc;
+
+ if (!dfsPath.exists())
+ throw new IOException("Directory " + path + " does not exist.");
+
+ if (mDfs.isContainer(path)) {
+ ContainerDescriptor dfsDir = (ContainerDescriptor) dfsPath;
+ Iterator<ElementDescriptor> paths = dfsDir.iterator();
+
+ while (paths.hasNext()) {
+ ElementDescriptor curElem = paths.next();
+
+ if (mDfs.isContainer(curElem.toString())) {
+ continue;
+ }
+
+ InputStream is = curElem.open();
+ while ((rc = is.read(buffer)) > 0) {
+ System.out.write(buffer, 0, rc);
+ }
+ is.close();
+ }
+ }
+ else {
+ InputStream is = dfsPath.open();
+ while ((rc = is.read(buffer)) > 0) {
System.out.write(buffer, 0, rc);
- is.close();
+ }
+ is.close();
}
- }
- else
- {
- FSDataInputStream is = mDfs.open(dfsPath);
- while ((rc = is.read(buffer)) > 0)
- System.out.write(buffer, 0, rc);
- is.close();
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to Cat: " + path, e);
}
}
protected void processCD(String path) throws IOException
{
- if (path == null)
- mDfs.setWorkingDirectory(new Path("/user/" + System.getProperty("user.name")));
- else
- {
- Path dfsDir = new Path(path);
-
- if (!mDfs.exists(dfsDir))
- throw new IOException("Directory " + path + " does not exist.");
-
- if (!mDfs.getFileStatus(dfsDir).isDir())
- throw new IOException(path + " is not a directory.");
+ ContainerDescriptor container;
- mDfs.setWorkingDirectory(dfsDir);
+ try {
+ if (path == null) {
+ container = mDfs.asContainer("/user/" + System.getProperty("user.name"));
+ mDfs.setActiveContainer(container);
+ }
+ else
+ {
+ container = mDfs.asContainer(path);
+
+ if (!container.exists()) {
+ throw new IOException("Directory " + path + " does not exist.");
+ }
+
+ if (!mDfs.isContainer(path)) {
+ throw new IOException(path + " is not a directory.");
+ }
+
+ mDfs.setActiveContainer(container);
+ }
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to change working directory to " +
+ ((path == null) ? ("/user/" + System.getProperty("user.name"))
+ : (path)),
+ e);
}
}
@@ -208,29 +248,57 @@
protected void processLS(String path) throws IOException
{
- Path dir;
- if(path == null)
- dir = mDfs.getWorkingDirectory();
- else
- {
- dir = new Path(path);
- if (!mDfs.exists(dir))
- throw new IOException("File or directory " + path + " does not exist.");
- }
+ try {
+ ElementDescriptor pathDescriptor;
+
+ if (path == null) {
+ pathDescriptor = mDfs.getActiveContainer();
+ }
+ else {
+ pathDescriptor = mDfs.asElement(path);
+ }
- FileStatus fileStat[] = mDfs.listStatus(dir);
- for (int j = 0; j < fileStat.length; j++)
- {
- if (fileStat[j].isDir())
- System.out.println(fileStat[j].getPath() + "\t<dir>");
- else
- System.out.println(fileStat[j].getPath() + "<r " + fileStat[j].getReplication() + ">\t" + fileStat[j].getLen());
- }
+ if (!pathDescriptor.exists()) {
+ throw new IOException("File or directory " + path + " does not exist.");
+ }
+
+ if (mDfs.isContainer(pathDescriptor.toString())) {
+ ContainerDescriptor container = (ContainerDescriptor) pathDescriptor;
+ Iterator<ElementDescriptor> elems = container.iterator();
+
+ while (elems.hasNext()) {
+ ElementDescriptor curElem = elems.next();
+
+ if (mDfs.isContainer(curElem.toString())) {
+ System.out.println(curElem.toString() + "\t<dir>");
+ }
+ else {
+ Properties config = curElem.getConfiguration();
+ Map<String, Object> stats = curElem.getStatistics();
+
+ String strReplication = config.getProperty(ElementDescriptor.BLOCK_REPLICATION_KEY);
+ String strLen = (String) stats.get(ElementDescriptor.LENGTH_KEY);
+
+ System.out.println(curElem.toString() + "<r " + strReplication + ">\t" + strLen);
+ }
+ }
+ }
+ else {
+ Map<String, Object> properties = pathDescriptor.getStatistics();
+ String strReplication = (String) properties.get(DataStorage.DEFAULT_REPLICATION_FACTOR_KEY);
+ String strLen = (String) properties.get(DataStorage.USED_BYTES_KEY);
+
+ System.out.println(pathDescriptor.toString() + "<r " + strReplication + ">\t" + strLen);
+ }
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to LS on " + path, e);
+ }
}
protected void processPWD() throws IOException
{
- System.out.println(mDfs.getWorkingDirectory());
+ System.out.println(mDfs.getActiveContainer().toString());
}
protected void printHelp()
@@ -253,32 +321,70 @@
protected void processMove(String src, String dst) throws IOException
{
- Path srcPath = new Path(src);
- Path dstPath = new Path(dst);
- if (!mDfs.exists(srcPath))
- throw new IOException("File or directory " + src + " does not exist.");
-
- {mDfs.rename(srcPath, dstPath);}
+ try {
+ ElementDescriptor srcPath = mDfs.asElement(src);
+ ElementDescriptor dstPath = mDfs.asElement(dst);
+
+ if (!srcPath.exists()) {
+ throw new IOException("File or directory " + src + " does not exist.");
+ }
+
+ srcPath.rename(dstPath);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to move " + src + " to " + dst, e);
+ }
}
protected void processCopy(String src, String dst) throws IOException
{
- FileUtil.copy(mDfs, new Path(src), mDfs, new Path(dst), false, mConf);
+ try {
+ ElementDescriptor srcPath = mDfs.asElement(src);
+ ElementDescriptor dstPath = mDfs.asElement(dst);
+
+ srcPath.copy(dstPath, mConf, false);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to copy " + src + " to " + dst, e);
+ }
}
protected void processCopyToLocal(String src, String dst) throws IOException
{
- mDfs.copyToLocalFile(new Path(src), new Path(dst));
+ try {
+ ElementDescriptor srcPath = mDfs.asElement(src);
+ ElementDescriptor dstPath = mLfs.asElement(dst);
+
+ srcPath.copy(dstPath, false);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to copy " + src + "to (locally) " + dst, e);
+ }
}
protected void processCopyFromLocal(String src, String dst) throws IOException
{
- mDfs.copyFromLocalFile(new Path(src), new Path(dst));
+ try {
+ ElementDescriptor srcPath = mLfs.asElement(src);
+ ElementDescriptor dstPath = mDfs.asElement(dst);
+
+ srcPath.copy(dstPath, false);
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to copy (loally) " + src + "to " + dst, e);
+ }
}
protected void processMkdir(String dir) throws IOException
{
- mDfs.mkdirs(new Path(dir));
+ try {
+ ContainerDescriptor dirDescriptor = mDfs.asContainer(dir);
+
+ dirDescriptor.create();
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to create dir: " + dir, e);
+ }
}
protected void processPig(String cmd) throws IOException
@@ -291,16 +397,24 @@
protected void processRemove(String path) throws IOException
{
- Path dfsPath = new Path(path);
- if (!mDfs.exists(dfsPath))
- throw new IOException("File or directory " + path + " does not exist.");
-
- mDfs.delete(dfsPath);
+ try {
+ ElementDescriptor dfsPath = mDfs.asElement(path);
+
+ if (!dfsPath.exists()) {
+ throw new IOException("File or directory " + path + " does not exist.");
+ }
+
+ dfsPath.delete();
+ }
+ catch (DataStorageException e) {
+ throw new IOException("Failed to get descriptor for " + path, e);
+ }
}
private PigServer mPigServer;
- private FileSystem mDfs;
- private Configuration mConf;
+ private DataStorage mDfs;
+ private DataStorage mLfs;
+ private Properties mConf;
private JobClient mJobClient;
private boolean mDone;
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java Thu Jan 31 19:11:28 2008
@@ -33,38 +33,8 @@
public class TestAlgebraicEval extends TestCase {
private String initString = "mapreduce";
-
- @Test
- public void testGroupCountWithMultipleFields() throws Exception {
- int LOOP_COUNT = 1024;
- PigServer pig = new PigServer(initString);
- File tmpFile = File.createTempFile("test", "txt");
- PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
- for(int i = 0; i < LOOP_COUNT; i++) {
- for(int j=0; j< LOOP_COUNT; j++) {
- ps.println(i + "\t" + i + "\t" + j%2);
- }
- }
- ps.close();
- String query = "myid = foreach (group (load 'file:" + tmpFile + "') all) generate group, COUNT($1) ;";
- System.out.println(query);
- pig.registerQuery(" a = group (load 'file:" + tmpFile + "') by ($0,$1);");
- pig.registerQuery("b = foreach a generate flatten(group), SUM($1.$2);");
- Iterator<Tuple> it = pig.openIterator("b");
- tmpFile.delete();
- int count = 0;
- while(it.hasNext()){
- int sum = it.next().getAtomField(2).numval().intValue();
- assertEquals(LOOP_COUNT/2, sum);
- count++;
- }
- assertEquals(count, LOOP_COUNT);
- }
-
-
-
@Test
- public void testSimpleCount() throws Exception {
+ public void testSimpleCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -84,7 +54,7 @@
}
@Test
- public void testGroupCount() throws Exception {
+ public void testGroupCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -103,10 +73,8 @@
assertEquals(count, (double)LOOP_COUNT);
}
-
-
@Test
- public void testGroupReorderCount() throws Exception {
+ public void testGroupReorderCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -128,7 +96,7 @@
@Test
- public void testGroupUniqueColumnCount() throws Exception {
+ public void testGroupUniqueColumnCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -156,7 +124,7 @@
}
@Test
- public void testGroupDuplicateColumnCount() throws Exception {
+ public void testGroupDuplicateColumnCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java Thu Jan 31 19:11:28 2008
@@ -456,7 +456,7 @@
}
@Test
- public void testShellFuncSingle() throws Exception {
+ public void testShellFuncSingle() throws Throwable {
//ShellBagEvalFunc func = new ShellBagEvalFunc("tr o 0");
PigServer pig = new PigServer(initString);
@@ -484,7 +484,7 @@
}
@Test
- public void testShellFuncMultiple() throws Exception {
+ public void testShellFuncMultiple() throws Throwable {
PigServer pig = new PigServer(initString);
final int numTimes = 100;
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java Thu Jan 31 19:11:28 2008
@@ -65,7 +65,7 @@
}
@Test
- public void testCompressed1() throws Exception {
+ public void testCompressed1() throws Throwable {
PigServer pig = new PigServer("mapreduce");
pig.registerQuery("A = foreach (cogroup (load 'file:"+gzFile+"') by $1, (load 'file:"+datFile + "') by $1) generate flatten( " + DIFF.class.getName() + "($1.$1,$2.$1)) ;");
Iterator it = pig.openIterator("A");
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java Thu Jan 31 19:11:28 2008
@@ -40,6 +40,8 @@
import org.apache.pig.data.*;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.PigFile;
+
+import org.apache.pig.backend.executionengine.ExecException;
import junit.framework.TestCase;
@@ -67,9 +69,9 @@
pw.close();
return f;
}
-
+
@Test
- public void testFunctionInsideFunction() throws Exception{
+ public void testFunctionInsideFunction() throws Throwable {
PigServer pigServer = new PigServer(initString);
File f1 = createFile(new String[]{"a:1","b:1","a:1"});
@@ -85,7 +87,7 @@
}
@Test
- public void testJoin() throws Exception{
+ public void testJoin() throws Throwable {
PigServer pigServer = new PigServer(initString);
File f1 = createFile(new String[]{"a:1","b:1","a:1"});
@@ -107,7 +109,7 @@
}
@Test
- public void testDriverMethod() throws Exception{
+ public void testDriverMethod() throws Throwable {
PigServer pigServer = new PigServer(initString);
File f = File.createTempFile("tmp", "");
PrintWriter pw = new PrintWriter(f);
@@ -130,7 +132,7 @@
@Test
- public void testMapLookup() throws IOException{
+ public void testMapLookup() throws Throwable {
PigServer pigServer = new PigServer(initString);
DataBag b = BagFactory.getInstance().newDefaultBag();
DataMap colors = new DataMap();
@@ -211,12 +213,9 @@
return sb.toString();
}
}
-
-
-
-
+
@Test
- public void testBagFunctionWithFlattening() throws Exception{
+ public void testBagFunctionWithFlattening() throws Throwable {
PigServer pigServer = new PigServer(initString);
File queryLogFile = createFile(
new String[]{
@@ -259,20 +258,17 @@
}
}
-
-
@Test
- public void testSort() throws Exception{
+ public void testSort() throws Throwable {
testSortDistinct(false);
}
-
@Test
- public void testDistinct() throws Exception{
+ public void testDistinct() throws Throwable {
testSortDistinct(true);
}
-
- private void testSortDistinct(boolean eliminateDuplicates) throws Exception{
+
+ private void testSortDistinct(boolean eliminateDuplicates) throws Throwable {
int LOOP_SIZE = 1024*16;
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java Thu Jan 31 19:11:28 2008
@@ -37,7 +37,7 @@
private String initString = "mapreduce";
@Test
- public void testNumericEq() throws Exception {
+ public void testNumericEq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -68,7 +68,7 @@
}
@Test
- public void testNumericNeq() throws Exception {
+ public void testNumericNeq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -95,7 +95,7 @@
}
@Test
- public void testNumericGt() throws Exception {
+ public void testNumericGt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -123,7 +123,7 @@
}
@Test
- public void testBinCond() throws Exception {
+ public void testBinCond() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -152,7 +152,7 @@
@Test
- public void testNestedBinCond() throws Exception {
+ public void testNestedBinCond() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -180,7 +180,7 @@
@Test
- public void testNumericLt() throws Exception {
+ public void testNumericLt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -210,7 +210,7 @@
@Test
- public void testNumericGte() throws Exception {
+ public void testNumericGte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -240,7 +240,7 @@
}
@Test
- public void testNumericLte() throws Exception {
+ public void testNumericLte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java Thu Jan 31 19:11:28 2008
@@ -36,7 +36,7 @@
private String initString = "mapreduce";
@Test
- public void testStringEq() throws Exception {
+ public void testStringEq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -64,7 +64,7 @@
}
@Test
- public void testStringNeq() throws Exception {
+ public void testStringNeq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -92,7 +92,7 @@
}
@Test
- public void testStringGt() throws Exception {
+ public void testStringGt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -122,7 +122,7 @@
@Test
- public void testStringGte() throws Exception {
+ public void testStringGte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -153,7 +153,7 @@
}
@Test
- public void testStringLt() throws Exception {
+ public void testStringLt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -182,7 +182,7 @@
}
@Test
- public void testStringLte() throws Exception {
+ public void testStringLte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
Added: incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=617338&view=auto
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java (added)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java Thu Jan 31 19:11:28 2008
@@ -0,0 +1,31 @@
+package org.apache.pig.test;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.grunt.Grunt;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+
+public class TestGrunt extends TestCase {
+
+
+ @Test
+ public void testCopyFromLocal() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "copyFromLocal /tmp/TestMe;";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java Thu Jan 31 19:11:28 2008
@@ -36,7 +36,7 @@
private String initString = "mapreduce";
@Test
- public void testAdd() throws Exception {
+ public void testAdd() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -58,7 +58,7 @@
}
@Test
- public void testSubtract() throws Exception {
+ public void testSubtract() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -79,7 +79,7 @@
}
@Test
- public void testMultiply() throws Exception {
+ public void testMultiply() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -101,7 +101,7 @@
}
@Test
- public void testDivide() throws Exception {
+ public void testDivide() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java Thu Jan 31 19:11:28 2008
@@ -30,10 +30,13 @@
import junit.framework.TestCase;
import java.io.File;
+import java.io.IOException;
import java.io.FileOutputStream;
import java.util.Iterator;
import java.util.Random;
+import org.apache.pig.backend.executionengine.ExecException;
+
import org.apache.hadoop.conf.Configuration;
//This class tests pig behavior with large file spanning multiple blocks along with group and count functions
//Order and Distinct functions are also tested.
@@ -84,7 +87,12 @@
dat.close();
- pig = new PigServer(initString);
+ try {
+ pig = new PigServer(initString);
+ }
+ catch (ExecException e) {
+ throw new IOException("Failed to create Pig server", e);
+ }
fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Thu Jan 31 19:11:28 2008
@@ -34,12 +34,13 @@
import org.apache.pig.PigServer.ExecType;
import org.apache.pig.impl.builtin.ShellBagEvalFunc;
import org.apache.pig.impl.io.BufferedPositionedInputStream;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
import org.apache.pig.impl.logicalLayer.LOCogroup;
import org.apache.pig.impl.logicalLayer.LOEval;
import org.apache.pig.impl.logicalLayer.LogicalOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
+
public class TestLogicalPlanBuilder extends junit.framework.TestCase {
@@ -198,13 +199,15 @@
public void testQuery18() {
String query = "FOREACH (group (load 'a') ALL PARALLEL 16) generate group;";
LogicalPlan lp = buildPlan(query);
- LogicalOperator lo = lp.getRoot().getInputs().get(0);
+ Map<OperatorKey, LogicalOperator> logicalOpTable = lp.getOpTable();
+ OperatorKey logicalKey = lp.getRoot();
+ LogicalOperator lo = logicalOpTable.get(logicalOpTable.get(logicalKey).getInputs().get(0));
+
if (lo instanceof LOCogroup) {
assertTrue(((LOCogroup) lo).getRequestedParallelism() == 16);
} else {
fail("Error: Unexpected Parse Tree output");
}
-
}
@@ -425,7 +428,7 @@
}
- public void testQueryFail44() {
+ public void testQueryFail44() throws Throwable {
PigServer pig = null;
try {
pig = new PigServer("local");
@@ -532,15 +535,18 @@
LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
try {
- LogicalPlan lp = builder.parse(query, queryResults);
- if (lp.getRoot() instanceof LOEval){
+ LogicalPlan lp = builder.parse("Test-Plan-Builder",
+ query,
+ aliases,
+ logicalOpTable);
+ if (logicalOpTable.get(lp.getRoot()) instanceof LOEval){
System.out.println(query);
- System.out.println(((LOEval)lp.getRoot()).getSpec());
+ System.out.println(((LOEval)logicalOpTable.get(lp.getRoot())).getSpec());
}
if (lp.getAlias()!=null){
- queryResults.put(lp.getAlias(), new IntermedResult(lp, pigContext));
+ aliases.put(lp.getAlias(), lp);
}
- //System.out.println(lp.root().outputSchema());
+
assertTrue(lp != null);
return lp;
} catch (IOException e) {
@@ -553,6 +559,6 @@
return null;
}
- Map<String, IntermedResult> queryResults = new HashMap<String, IntermedResult>();
-
+ Map<String, LogicalPlan> aliases = new HashMap<String, LogicalPlan>();
+ Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java Thu Jan 31 19:11:28 2008
@@ -29,7 +29,6 @@
import junit.framework.TestCase;
-import org.apache.hadoop.fs.Path;
import org.junit.Test;
import org.apache.pig.EvalFunc;
@@ -40,18 +39,18 @@
import org.apache.pig.builtin.COUNT;
import org.apache.pig.data.DataAtom;
import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Datum;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.BufferedPositionedInputStream;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.backend.datastorage.ElementDescriptor;
public class TestMapReduce extends TestCase {
private String initString = "mapreduce";
-
+
@Test
- public void testBigGroupAll() throws Exception {
+ public void testBigGroupAll() throws Throwable {
int LOOP_COUNT = 4*1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -119,7 +118,7 @@
}
}
@Test
- public void testStoreFunction() throws IOException {
+ public void testStoreFunction() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", ".txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -146,7 +145,7 @@
pig.deleteFile("frog");
}
@Test
- public void testQualifiedFuncions() throws IOException {
+ public void testQualifiedFuncions() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", ".txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -171,7 +170,7 @@
}
@Test
- public void testDefinedFunctions() throws IOException {
+ public void testDefinedFunctions() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", ".txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -195,9 +194,9 @@
}
assertEquals(count, MyStorage.COUNT);
}
-
+
@Test
- public void testPigServer() throws IOException {
+ public void testPigServer() throws Throwable {
System.out.println("creating pig server");
PigContext pigContext = new PigContext(ExecType.MAPREDUCE);
PigServer pig = new PigServer(pigContext);
@@ -206,24 +205,12 @@
assertTrue(capacity > 0);
String sampleFileName = "/tmp/fileTest";
if (!pig.existsFile(sampleFileName)) {
- OutputStream os = pigContext.getDfs().create(new Path(sampleFileName));
+ ElementDescriptor path = pigContext.getDfs().asElement(sampleFileName);
+ OutputStream os = path.create();
os.write("Ben was here!".getBytes());
os.close();
}
long length = pig.fileSize(sampleFileName);
assertTrue(length > 0);
}
-
- @Test
- public void testCreateNewRelation() throws IOException {
- System.out.println("creating pig server");
- PigServer pig = new PigServer(initString);
- pig.deleteFile("/tmp/test_createNewRelation");
- System.out.println("testing create new relation");
- pig.newRelation("new_rel");
- pig.insertTuple("new_rel", new Tuple("hello"));
- pig.store("new_rel", "/tmp/test_createNewRelation");
- assertTrue(pig.existsFile("/tmp/test_createNewRelation"));
- }
-
}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java Thu Jan 31 19:11:28 2008
@@ -38,7 +38,7 @@
private PigServer pig;
private File tmpFile;
- public TestOrderBy() throws Exception {
+ public TestOrderBy() throws Throwable {
DecimalFormat myFormatter = new DecimalFormat("0000000");
for (int i = 0; i < DATALEN; i++) {
DATA[0][i] = myFormatter.format(i);
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPi.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPi.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPi.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPi.java Thu Jan 31 19:11:28 2008
@@ -24,12 +24,15 @@
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.backend.executionengine.ExecException;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import junit.framework.TestCase;
import java.io.File;
+import java.io.IOException;
import java.io.FileOutputStream;
import java.util.Iterator;
import java.util.Random;
@@ -90,8 +93,14 @@
dat.close();
- pig = new PigServer(initString);
- fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
+ try {
+ pig = new PigServer(initString);
+ }
+ catch (ExecException e) {
+ throw new IOException("Failed to create Pig Server", e);
+ }
+
+ fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
datFile.delete();
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java Thu Jan 31 19:11:28 2008
@@ -178,7 +178,7 @@
}
- public void testLocalStore() throws Exception{
+ public void testLocalStore() throws Throwable {
PigServer pig = new PigServer("local");
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Thu Jan 31 19:11:28 2008
@@ -18,8 +18,15 @@
public class TestPigServer extends TestCase {
- private PigServer pig = new PigServer();
-
+ private PigServer pig = null;
+
+ private void initPigServer() throws Throwable {
+ if (pig == null) {
+ pig = new PigServer();
+ }
+ }
+
+
private final static String FILE_SEPARATOR = System.getProperty("file.separator");
// make sure that name is included or not (depending on flag "included")
@@ -79,12 +86,13 @@
* The jar file to register is not present
*/
@Test
- public void testRegisterJarFileNotPresent() throws Exception {
+ public void testRegisterJarFileNotPresent() throws Throwable {
// resister a jar file that does not exist
String jarName = "BadFileNameTestJarNotPresent.jar";
// jar name is not present to start with
+ initPigServer();
verifyStringContained(pig.getPigContext().extraJars, jarName, false);
boolean exceptionRaised = false;
@@ -103,13 +111,15 @@
* in this case name of jar file is relative to current working dir
*/
@Test
- public void testRegisterJarLocalDir() throws Exception {
+ public void testRegisterJarLocalDir() throws Throwable {
String dir1 = "test1_register_jar_local";
String dir2 = "test2_register_jar_local";
String jarLocation = dir1 + FILE_SEPARATOR +
dir2 + FILE_SEPARATOR;
String jarName = "TestRegisterJarLocal.jar";
+ initPigServer();
+
createFakeJarFile(jarLocation, jarName);
verifyStringContained(pig.getPigContext().extraJars, jarName, false);
@@ -136,7 +146,7 @@
* only one of them is registered.
*/
@Test
- public void testRegisterJarFromResources () throws Exception {
+ public void testRegisterJarFromResources () throws Throwable {
String dir = "test_register_jar_res_dir";
String subDir1 = "test_register_jar_res_sub_dir1";
String subDir2 = "test_register_jar_res_sub_dir2";
@@ -144,6 +154,8 @@
String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
+ initPigServer();
+
createFakeJarFile(jarLocation1, jarName);
createFakeJarFile(jarLocation2, jarName);
@@ -176,13 +188,15 @@
* @throws Exception
*/
@Test
- public void testRegisterJarResourceInJar() throws Exception {
+ public void testRegisterJarResourceInJar() throws Throwable {
String dir = "test_register_jar_res_in_jar";
String subDir = "sub_dir";
String jarName = "TestRegisterJarNonEmpty.jar";
String className = "TestRegisterJar";
String javaSrc = "package " + subDir + "; class " + className + " { }";
+ initPigServer();
+
// create dirs
(new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java Thu Jan 31 19:11:28 2008
@@ -20,6 +20,7 @@
import java.io.BufferedWriter;
import java.io.File;
+import java.io.IOException;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Iterator;
@@ -28,6 +29,7 @@
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
+import org.apache.pig.backend.executionengine.ExecException;
import junit.framework.TestCase;
@@ -37,9 +39,14 @@
@Override
protected void setUp() throws Exception {
super.setUp();
- pig = new PigServer();
+
+ try {
+ pig = new PigServer();
+ }
+ catch (ExecException e) {
+ throw new IOException("Failed to create Pig Server", e);
+ }
}
-
@Test
public void testLongEvalSpec() throws Exception{
File f = File.createTempFile("tmp", "");
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=617338&r1=617337&r2=617338&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestStore.java Thu Jan 31 19:11:28 2008
@@ -18,12 +18,14 @@
package org.apache.pig.test;
import java.io.File;
+import java.io.IOException;
import java.io.PrintWriter;
import java.util.Iterator;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.backend.executionengine.ExecException;
import junit.framework.TestCase;
@@ -35,7 +37,7 @@
String fileName;
String tmpFile1, tmpFile2;
PigServer pig;
-
+
public void testSingleStore() throws Exception{
pig.registerQuery("A = load " + fileName + ";");
@@ -103,8 +105,14 @@
for (int i=0;i<LOOP_COUNT; i++){
pw.println(i + "\t" + i);
}
- pw.close();
- pig = new PigServer(initString);
+ pw.close();
+ try {
+ pig = new PigServer(initString);
+ }
+ catch (ExecException e) {
+ throw new IOException("Failed to create Pig Server", e);
+ }
+
fileName = "'" + FileLocalizer.hadoopify(f.toString(), pig.getPigContext()) + "'";
tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
tmpFile2 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";