You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/08/04 19:12:01 UTC
svn commit: r800877 - in /hadoop/pig/branches/branch-0.3: ./
src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
Author: pradeepkth
Date: Tue Aug 4 17:12:01 2009
New Revision: 800877
URL: http://svn.apache.org/viewvc?rev=800877&view=rev
Log:
PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to serialized PigContext (pradeepkth)
Modified:
hadoop/pig/branches/branch-0.3/CHANGES.txt
hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
Modified: hadoop/pig/branches/branch-0.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/CHANGES.txt?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.3/CHANGES.txt Tue Aug 4 17:12:01 2009
@@ -22,6 +22,9 @@
BUG FIXES
+PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to
+serialized PigContext (pradeepkth)
+
PIG-878: Pig is returning too many blocks in the input split. (arunc via gates).
Release 0.3.0 - 06/16/09
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/ExecType.java Tue Aug 4 17:12:01 2009
@@ -18,10 +18,12 @@
package org.apache.pig;
+import java.io.Serializable;
+
/**
* The type of query execution
*/
-public enum ExecType {
+public enum ExecType implements Serializable {
/**
* Run everything on the local machine
*/
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java Tue Aug 4 17:12:01 2009
@@ -233,7 +233,7 @@
}
Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName());
for (Slice split : pigs) {
- splits.add(new SliceWrapper(split, pigContext, i, fs, inpTargets.get(i)));
+ splits.add(new SliceWrapper(split, pigContext.getExecType(), i, fs, inpTargets.get(i)));
}
} catch (ExecException ee) {
throw ee;
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java?rev=800877&r1=800876&r2=800877&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SliceWrapper.java Tue Aug 4 17:12:01 2009
@@ -60,7 +60,7 @@
public class SliceWrapper implements InputSplit {
private int index;
- private PigContext pigContext;
+ private ExecType execType;
private Slice wrapped;
private transient FileSystem fs;// transient so it isn't serialized
private transient JobConf lastConf;
@@ -70,9 +70,9 @@
// for deserialization
}
- public SliceWrapper(Slice slice, PigContext context, int index, FileSystem fs, ArrayList<OperatorKey> targetOps) {
+ public SliceWrapper(Slice slice, ExecType execType, int index, FileSystem fs, ArrayList<OperatorKey> targetOps) {
this.wrapped = slice;
- this.pigContext = context;
+ this.execType = execType;
this.index = index;
this.fs = fs;
this.targetOps = targetOps;
@@ -122,7 +122,7 @@
DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
// if the execution is against Mapred DFS, set
// working dir to /user/<userid>
- if(pigContext.getExecType() == ExecType.MAPREDUCE)
+ if(execType == ExecType.MAPREDUCE)
store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
wrapped.init(store);
@@ -165,7 +165,7 @@
}
public void readFields(DataInput is) throws IOException {
- pigContext = (PigContext) readObject(is);
+ execType = (ExecType) readObject(is);
targetOps = (ArrayList<OperatorKey>) readObject(is);
index = is.readInt();
wrapped = (Slice) readObject(is);
@@ -192,7 +192,7 @@
}
public void write(DataOutput os) throws IOException {
- writeObject(pigContext, os);
+ writeObject(execType, os);
writeObject(targetOps, os);
os.writeInt(index);
writeObject(wrapped, os);