Review Board 1.7.22


Allow UDF's to flatten themselves (no ws)

Review Request #9529 - Created Feb. 20, 2013 and updated

Jonathan Coveney
PIG-3010
Reviewers
pig
dvryaboy
pig-git
https://issues.apache.org/jira/browse/PIG-3010

 

Diff revision 2 (Latest)

1 2
1 2

  1. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java: Loading...
  2. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java: Loading...
  3. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java: Loading...
  4. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java: Loading...
  5. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java: Loading...
  6. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POOptimizedForEach.java: Loading...
  7. src/org/apache/pig/builtin/FlattenOutput.java: Loading...
  8. src/org/apache/pig/builtin/UdfFlatten.java: Loading...
  9. src/org/apache/pig/newplan/logical/Util.java: Loading...
  10. src/org/apache/pig/newplan/logical/relational/LOGenerate.java: Loading...
  11. src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java: Loading...
  12. src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java: Loading...
  13. src/org/apache/pig/newplan/logical/rules/ColumnPruneVisitor.java: Loading...
  14. src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java: Loading...
  15. src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java: Loading...
  16. src/org/apache/pig/newplan/logical/rules/MergeForEach.java: Loading...
  17. src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java: Loading...
  18. src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java: Loading...
  19. src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java: Loading...
  20. src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java: Loading...
  21. src/org/apache/pig/newplan/logical/visitor/ProjectStarExpander.java: Loading...
  22. src/org/apache/pig/newplan/logical/visitor/TypeCheckingRelVisitor.java: Loading...
  23. src/org/apache/pig/parser/LogicalPlanBuilder.java: Loading...
  24. src/org/apache/pig/parser/LogicalPlanGenerator.g: Loading...
  25. test/org/apache/pig/test/OptimizeLimitPlanPrinter.java: Loading...
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Revision eddffad New Change
[20] 83 lines
[+20]
84
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
84
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
85
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
85
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
86
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
86
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
87
import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
87
import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
88
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
88
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;

    
   
89
import org.apache.pig.builtin.FlattenOutput.FlattenStates;
89
import org.apache.pig.data.DataType;
90
import org.apache.pig.data.DataType;
90
import org.apache.pig.impl.PigContext;
91
import org.apache.pig.impl.PigContext;
91
import org.apache.pig.impl.builtin.DefaultIndexableLoader;
92
import org.apache.pig.impl.builtin.DefaultIndexableLoader;
92
import org.apache.pig.impl.builtin.FindQuantiles;
93
import org.apache.pig.impl.builtin.FindQuantiles;
93
import org.apache.pig.impl.builtin.GetMemNumRows;
94
import org.apache.pig.impl.builtin.GetMemNumRows;
[+20] [20] 2571 lines
[+20] [+] public void visitMROp(MapReduceOper mr) throws VisitorException {
2665
            boolean allSimple = true;
2666
            boolean allSimple = true;
2666
            if (op instanceof POForEach)
2667
            if (op instanceof POForEach)
2667
            {
2668
            {
2668
                POForEach forEach = (POForEach)op;
2669
                POForEach forEach = (POForEach)op;
2669
                List<PhysicalPlan> planList = forEach.getInputPlans();
2670
                List<PhysicalPlan> planList = forEach.getInputPlans();
2670
                List<Boolean> flatten = forEach.getToBeFlattened();
2671
                List<FlattenStates> flatten = forEach.getToBeFlattened();
2671
                POProject projOfLastInput = null;
2672
                POProject projOfLastInput = null;
2672
                int i = 0;
2673
                int i = 0;
2673
                // check all nested foreach plans
2674
                // check all nested foreach plans
2674
                // 1. If it is simple projection
2675
                // 1. If it is simple projection
2675
                // 2. If last input is all flattened
2676
                // 2. If last input is all flattened
[+20] [20] 30 lines
[+20] public void visitMROp(MapReduceOper mr) throws VisitorException {
2706
                                break;
2707
                                break;
2707
                            }
2708
                            }
2708
                            projOfLastInput = proj;
2709
                            projOfLastInput = proj;
2709
                            // make sure the project is on a bag which needs to be
2710
                            // make sure the project is on a bag which needs to be
2710
                            // flattened
2711
                            // flattened
2711
                            if (!flatten.get(i) || proj.getResultType() != DataType.BAG)
2712
                            if (!flatten.get(i).shouldFlatten() || proj.getResultType() != DataType.BAG)
2712
                            {
2713
                            {
2713
                                lastInputFlattened = false;
2714
                                lastInputFlattened = false;
2714
                                break;
2715
                                break;
2715
                            }
2716
                            }
2716
                        }
2717
                        }
[+20] [20] 108 lines
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java
Revision 93de6d5 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
Revision a4abdd8 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
Revision 14598cc New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
Revision 7e357ec New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POOptimizedForEach.java
Revision 91b3f00 New Change
 
src/org/apache/pig/builtin/FlattenOutput.java
New File
 
src/org/apache/pig/builtin/UdfFlatten.java
New File
 
src/org/apache/pig/newplan/logical/Util.java
Revision c992550 New Change
 
src/org/apache/pig/newplan/logical/relational/LOGenerate.java
Revision 383ba15 New Change
 
src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
Revision 5f33b07 New Change
 
src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
Revision 19e7628 New Change
 
src/org/apache/pig/newplan/logical/rules/ColumnPruneVisitor.java
Revision a458173 New Change
 
src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java
Revision 7091587 New Change
 
src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java
Revision c32f9b3 New Change
 
src/org/apache/pig/newplan/logical/rules/MergeForEach.java
Revision 8415a5c New Change
 
src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java
Revision 4356130 New Change
 
src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
Revision aedb075 New Change
 
src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java
Revision 35dcfce New Change
 
src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
Revision 4235d82 New Change
 
src/org/apache/pig/newplan/logical/visitor/ProjectStarExpander.java
Revision 135de1c New Change
 
src/org/apache/pig/newplan/logical/visitor/TypeCheckingRelVisitor.java
Revision 466ed1a New Change
 
src/org/apache/pig/parser/LogicalPlanBuilder.java
Revision b705686 New Change
 
src/org/apache/pig/parser/LogicalPlanGenerator.g
Revision 01dc570 New Change
 
test/org/apache/pig/test/OptimizeLimitPlanPrinter.java
Revision eadc1d5 New Change
 
  1. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java: Loading...
  2. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java: Loading...
  3. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java: Loading...
  4. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java: Loading...
  5. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java: Loading...
  6. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POOptimizedForEach.java: Loading...
  7. src/org/apache/pig/builtin/FlattenOutput.java: Loading...
  8. src/org/apache/pig/builtin/UdfFlatten.java: Loading...
  9. src/org/apache/pig/newplan/logical/Util.java: Loading...
  10. src/org/apache/pig/newplan/logical/relational/LOGenerate.java: Loading...
  11. src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java: Loading...
  12. src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java: Loading...
  13. src/org/apache/pig/newplan/logical/rules/ColumnPruneVisitor.java: Loading...
  14. src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java: Loading...
  15. src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java: Loading...
  16. src/org/apache/pig/newplan/logical/rules/MergeForEach.java: Loading...
  17. src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java: Loading...
  18. src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java: Loading...
  19. src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java: Loading...
  20. src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java: Loading...
  21. src/org/apache/pig/newplan/logical/visitor/ProjectStarExpander.java: Loading...
  22. src/org/apache/pig/newplan/logical/visitor/TypeCheckingRelVisitor.java: Loading...
  23. src/org/apache/pig/parser/LogicalPlanBuilder.java: Loading...
  24. src/org/apache/pig/parser/LogicalPlanGenerator.g: Loading...
  25. test/org/apache/pig/test/OptimizeLimitPlanPrinter.java: Loading...