Review Board 1.7.22


PIG-3591: Refactor POPackage

Review Request #15881 - Created Nov. 27, 2013 and updated

Mark Wagner
PIG-3591
Reviewers
pig
cheolsoo
pig-git
Separate "packaging" logic from "shuffle handling" logic. This moves the packaging logic to a new class "Packager", which is extended by CombinePackager, LitePackager, MultiQueryPackager, and JoinPackager.

This is not finished. Known problem are illustrate and streaming the last input are not implemented.
ant test-commit passes, with the exception of TestExampleGenerator which is caused by the illustrate work not being finished. I also ran some manual scripts, which worked.

Diff revision 5 (Latest)

1 2 3 4 5
1 2 3 4 5

  1. src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java: Loading...
  2. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java: Loading...
  3. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/CombinerOptimizer.java: Loading...
  4. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java: Loading...
  5. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java: Loading...
  6. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java: Loading...
  7. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java: Loading...
  8. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java: Loading...
  9. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java: Loading...
  10. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java: Loading...
  11. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java: Loading...
  12. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java: Loading...
  13. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/POPackageAnnotator.java: Loading...
  14. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java: Loading...
  15. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PlanPrinter.java: Loading...
  16. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/XMLPhysicalPlanPrinter.java: Loading...
  17. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/CombinerPackager.java: Loading...
  18. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/JoinPackager.java: Loading...
  19. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/LitePackager.java: Loading...
  20. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/MultiQueryPackager.java: Loading...
This diff has been split across 3 pages: 1 2 3 >
src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java
Revision d801f6f New Change
[20] 29 lines
[+20]
30
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PhyPlanSetter;
30
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PhyPlanSetter;
31
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
31
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
32
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
32
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
33
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
33
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
34
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
34
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
35
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCombinerPackage;

   
36
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter;
35
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter;
37
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCross;
36
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCross;
38
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
37
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
39
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
38
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
40
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFRJoin;
39
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFRJoin;
41
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
40
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
42
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POJoinPackage;

   
43
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
41
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
44
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
42
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
45
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeCogroup;
43
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeCogroup;
46
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin;
44
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin;
47
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMultiQueryPackage;

   
48
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PONative;
45
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PONative;
49
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POOptimizedForEach;
46
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POOptimizedForEach;
50
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
47
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
51
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg;
48
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg;
52
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartitionRearrange;
49
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartitionRearrange;
[+20] [20] 176 lines
[+20] [+] public void visitGlobalRearrange(POGlobalRearrange gr) throws VisitorException {
229
        public void visitPackage(POPackage pkg) throws VisitorException {
226
        public void visitPackage(POPackage pkg) throws VisitorException {
230
            planFetchable = false;
227
            planFetchable = false;
231
        }
228
        }
232

    
   
229

   
233
        @Override
230
        @Override
234
        public void visitCombinerPackage(POCombinerPackage pkg) throws VisitorException {

   
235
            planFetchable = false;

   
236
        }

   
237

    
   

   
238
        @Override

   
239
        public void visitMultiQueryPackage(POMultiQueryPackage pkg) throws VisitorException {

   
240
            planFetchable = false;

   
241
        }

   
242

    
   

   
243
        @Override

   
244
        public void visitSplit(POSplit spl) throws VisitorException {
231
        public void visitSplit(POSplit spl) throws VisitorException {
245
            planFetchable = false;
232
            planFetchable = false;
246
        }
233
        }
247

    
   
234

   
248
        @Override
235
        @Override
[+20] [20] 20 lines
[+20] [+] public void visitDistinct(PODistinct distinct) throws VisitorException {
269
        public void visitSort(POSort sort) throws VisitorException {
256
        public void visitSort(POSort sort) throws VisitorException {
270
            planFetchable = false;
257
            planFetchable = false;
271
        }
258
        }
272

    
   
259

   
273
        @Override
260
        @Override
274
        public void visitJoinPackage(POJoinPackage joinPackage) throws VisitorException {

   
275
            planFetchable = false;

   
276
        }

   
277

    
   

   
278
        @Override

   
279
        public void visitCross(POCross cross) throws VisitorException {
261
        public void visitCross(POCross cross) throws VisitorException {
280
            planFetchable = false;
262
            planFetchable = false;
281
        }
263
        }
282

    
   
264

   
283
        @Override
265
        @Override
[+20] [20] 54 lines
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java
Revision 3638b5c New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/CombinerOptimizer.java
Revision 18a382b New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Revision b7ce7e2 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Revision 5dddab7 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java
Revision 93de6d5 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Revision d3ebeb3 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
Revision 64f0ee1 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
Revision 933363d New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
Revision 773a22c New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
Revision eea5ce3 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
Revision 54740a0 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/POPackageAnnotator.java
Revision 47137d5 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
Revision abb16ff New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PlanPrinter.java
Revision ff82801 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/XMLPhysicalPlanPrinter.java
Revision 892c26f New Change
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/CombinerPackager.java
New File
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/JoinPackager.java
New File
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/LitePackager.java
New File
 
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/MultiQueryPackager.java
New File
 
  1. src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java: Loading...
  2. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/AccumulatorOptimizer.java: Loading...
  3. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/CombinerOptimizer.java: Loading...
  4. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java: Loading...
  5. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java: Loading...
  6. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRUtil.java: Loading...
  7. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java: Loading...
  8. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java: Loading...
  9. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java: Loading...
  10. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java: Loading...
  11. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java: Loading...
  12. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java: Loading...
  13. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/POPackageAnnotator.java: Loading...
  14. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java: Loading...
  15. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PlanPrinter.java: Loading...
  16. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/XMLPhysicalPlanPrinter.java: Loading...
  17. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/CombinerPackager.java: Loading...
  18. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/JoinPackager.java: Loading...
  19. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/LitePackager.java: Loading...
  20. src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/MultiQueryPackager.java: Loading...
This diff has been split across 3 pages: 1 2 3 >