You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pig.apache.org by Aniket Mokashi <an...@gmail.com> on 2013/10/23 00:36:04 UTC
Re: svn commit: r1531441 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
test/org/apache/pig/test/TestJobSubmission.java
I don't see PIG-3512 in CHANGES.txt, is that expected?
On Fri, Oct 11, 2013 at 2:54 PM, <da...@apache.org> wrote:
> Author: daijy
> Date: Fri Oct 11 21:54:23 2013
> New Revision: 1531441
>
> URL: http://svn.apache.org/r1531441
> Log:
> PIG-3512: Reducer estimater is broken by PIG-3497
>
> Modified:
> pig/trunk/CHANGES.txt
>
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
>
> Modified: pig/trunk/CHANGES.txt
> URL:
> http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1531441&r1=1531440&r2=1531441&view=diff
>
> ==============================================================================
> --- pig/trunk/CHANGES.txt (original)
> +++ pig/trunk/CHANGES.txt Fri Oct 11 21:54:23 2013
> @@ -46,7 +46,13 @@ PIG-3469: Skewed join can cause unrecove
>
> PIG-3496: Propagate HBase 0.95 jars to the backend (Jarek Jarcec Cecho
> via xuefuz)
>
> -Release 0.12.0 (unreleased changes)
> +Release 0.12.1 (unreleased changes)
> +
> +BUG FIXES
> +
> +PIG-3510: New filter extractor fails with more than one filter statement
> (aniket486 via cheolsoo)
> +
> +Release 0.12.0
>
> INCOMPATIBLE CHANGES
>
> @@ -277,8 +283,6 @@ PIG-3013: BinInterSedes improve chararra
>
> BUG FIXES
>
> -PIG-3510: New filter extractor fails with more than one filter statement
> (aniket486 via cheolsoo)
> -
> PIG-3504: Fix e2e Describe_cmdline_12 (cheolsoo via daijy)
>
> PIG-3128: Document the BigInteger and BigDecimal data type (daijy via
> cheolsoo)
>
> Modified:
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> URL:
> http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1531441&r1=1531440&r2=1531441&view=diff
>
> ==============================================================================
> ---
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> (original)
> +++
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> Fri Oct 11 21:54:23 2013
> @@ -478,6 +478,11 @@ public class JobControlCompiler{
> inp.add(ld.getLFile());
> }
> }
> +
> + if(!mro.reducePlan.isEmpty()){
> + log.info("Reduce phase detected, estimating # of
> required reducers.");
> + adjustNumReducers(plan, mro, nwJob);
> + }
>
> if(lds!=null && lds.size()>0){
> for (POLoad ld : lds) {
> @@ -656,7 +661,6 @@ public class JobControlCompiler{
> POPackage pack = null;
> if(mro.reducePlan.isEmpty()){
> //MapOnly Job
> - log.info("Map only job, skipping reducer estimation");
> nwJob.setMapperClass(PigMapOnly.Map.class);
> nwJob.setNumReduceTasks(0);
> if(!pigContext.inIllustrator)
> @@ -670,11 +674,6 @@ public class JobControlCompiler{
> }
> else{
> //Map Reduce Job
> -
> - // Estimate the number of reducers
> - log.info("Reduce phase detected, estimating # of
> required reducers.");
> - adjustNumReducers(plan, mro, nwJob);
> -
> //Process the POPackage operator and remove it from the
> reduce plan
> if(!mro.combinePlan.isEmpty()){
> POPackage combPack =
> (POPackage)mro.combinePlan.getRoots().get(0);
>
> Modified: pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
> URL:
> http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJobSubmission.java?rev=1531441&r1=1531440&r2=1531441&view=diff
>
> ==============================================================================
> --- pig/trunk/test/org/apache/pig/test/TestJobSubmission.java (original)
> +++ pig/trunk/test/org/apache/pig/test/TestJobSubmission.java Fri Oct 11
> 21:54:23 2013
> @@ -698,7 +698,7 @@ public class TestJobSubmission {
> Util.copyFromLocalToCluster(cluster,
> "test/org/apache/pig/test/data/passwd", ((POLoad)
> sort.mapPlan.getRoots().get(0)).getLFile().getFileName());
>
> //First job is just foreach with projection, mapper-only job, so
> estimate gets ignored
> - Util.assertParallelValues(-1, -1, reducer, 0,
> jobControl.getWaitingJobs().get(0).getJobConf());
> + Util.assertParallelValues(-1, -1, -1, 0,
> jobControl.getWaitingJobs().get(0).getJobConf());
>
> jcc.updateMROpPlan(jobControl.getReadyJobs());
> jobControl = jcc.compile(mrPlan, query);
>
>
>
--
"...:::Aniket:::... Quetzalco@tl"