Home | About | Sematext search-lucene.com search-hadoop.com
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB
 Search Hadoop and all its subprojects:

Switch to Threaded View
Pig >> mail # dev >> Re: svn commit: r1531441 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java test/org/apache/pig/test/TestJobSubmission.java


Copy link to this message
-
Re: svn commit: r1531441 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java test/org/apache/pig/test/TestJobSubmission.java
I don't see PIG-3512 in CHANGES.txt, is that expected?
On Fri, Oct 11, 2013 at 2:54 PM, <[EMAIL PROTECTED]> wrote:

> Author: daijy
> Date: Fri Oct 11 21:54:23 2013
> New Revision: 1531441
>
> URL: http://svn.apache.org/r1531441
> Log:
> PIG-3512: Reducer estimater is broken by PIG-3497
>
> Modified:
>     pig/trunk/CHANGES.txt
>
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
>     pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
>
> Modified: pig/trunk/CHANGES.txt
> URL:
> http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1531441&r1=1531440&r2=1531441&view=diff
>
> =============================================================================> --- pig/trunk/CHANGES.txt (original)
> +++ pig/trunk/CHANGES.txt Fri Oct 11 21:54:23 2013
> @@ -46,7 +46,13 @@ PIG-3469: Skewed join can cause unrecove
>
>  PIG-3496: Propagate HBase 0.95 jars to the backend (Jarek Jarcec Cecho
>  via xuefuz)
>
> -Release 0.12.0 (unreleased changes)
> +Release 0.12.1 (unreleased changes)
> +
> +BUG FIXES
> +
> +PIG-3510: New filter extractor fails with more than one filter statement
> (aniket486 via cheolsoo)
> +
> +Release 0.12.0
>
>  INCOMPATIBLE CHANGES
>
> @@ -277,8 +283,6 @@ PIG-3013: BinInterSedes improve chararra
>
>  BUG FIXES
>
> -PIG-3510: New filter extractor fails with more than one filter statement
> (aniket486 via cheolsoo)
> -
>  PIG-3504: Fix e2e Describe_cmdline_12 (cheolsoo via daijy)
>
>  PIG-3128: Document the BigInteger and BigDecimal data type (daijy via
> cheolsoo)
>
> Modified:
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> URL:
> http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1531441&r1=1531440&r2=1531441&view=diff
>
> =============================================================================> ---
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> (original)
> +++
> pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
> Fri Oct 11 21:54:23 2013
> @@ -478,6 +478,11 @@ public class JobControlCompiler{
>                      inp.add(ld.getLFile());
>                  }
>              }
> +
> +            if(!mro.reducePlan.isEmpty()){
> +                log.info("Reduce phase detected, estimating # of
> required reducers.");
> +                adjustNumReducers(plan, mro, nwJob);
> +            }
>
>              if(lds!=null && lds.size()>0){
>                for (POLoad ld : lds) {
> @@ -656,7 +661,6 @@ public class JobControlCompiler{
>              POPackage pack = null;
>              if(mro.reducePlan.isEmpty()){
>                  //MapOnly Job
> -                log.info("Map only job, skipping reducer estimation");
>                  nwJob.setMapperClass(PigMapOnly.Map.class);
>                  nwJob.setNumReduceTasks(0);
>                  if(!pigContext.inIllustrator)
> @@ -670,11 +674,6 @@ public class JobControlCompiler{
>              }
>              else{
>                  //Map Reduce Job
> -
> -                // Estimate the number of reducers
> -                log.info("Reduce phase detected, estimating # of
> required reducers.");
> -                adjustNumReducers(plan, mro, nwJob);
> -
>                  //Process the POPackage operator and remove it from the
> reduce plan
>                  if(!mro.combinePlan.isEmpty()){
>                      POPackage combPack > (POPackage)mro.combinePlan.getRoots().get(0);
>
> Modified: pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
> URL:
> http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJobSubmission.java?rev=1531441&r1=1531440&r2=1531441&view=diff
>
> =============================================================================> --- pig/trunk/test/org/apache/pig/test/TestJobSubmission.java (original)
> +++ pig/trunk/test/org/apache/pig/test/TestJobSubmission.java Fri Oct 11
"...:::Aniket:::... Quetzalco@tl"
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB