Review Board 1.7.22


Pig creates too many instances of Load and Store Funcs. It should be 1 in the front-end and 1 in the backend

Review Request #5159 - Created May 18, 2012 and submitted

Julien Le Dem
PIG-2699
Reviewers
pig
pig
Pig creates too many instances of Load and Store Funcs. It should be 1 in the front-end and 1 in the backend
test-commit + new test

Diff revision 2 (Latest)

1 2
1 2

  1. /trunk/src/org/apache/pig/PigServer.java: Loading...
  2. /trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java: Loading...
  3. /trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java: Loading...
  4. /trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java: Loading...
  5. /trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java: Loading...
  6. /trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java: Loading...
  7. /trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java: Loading...
  8. /trunk/src/org/apache/pig/newplan/logical/rules/LoadStoreFuncDupSignatureValidator.java: Loading...
  9. /trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java: Loading...
  10. /trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java: Loading...
  11. /trunk/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java: Loading...
  12. /trunk/src/org/apache/pig/parser/FunctionType.java: Loading...
  13. /trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java: Loading...
  14. /trunk/src/org/apache/pig/parser/QueryParserUtils.java: Loading...
  15. /trunk/test/org/apache/pig/TestLoadStoreFuncLifeCycle.java: Loading...
  16. /trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java: Loading...
  17. /trunk/test/org/apache/pig/parser/TestScalarVisitor.java: Loading...
  18. /trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java: Loading...
  19. /trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java: Loading...
  20. /trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java: Loading...
  21. /trunk/test/org/apache/pig/test/TestMRCompiler.java: Loading...
  22. /trunk/test/org/apache/pig/test/TestNewPlanFilterRule.java: Loading...
  23. /trunk/test/org/apache/pig/test/TestNewPlanListener.java: Loading...
  24. /trunk/test/org/apache/pig/test/TestNewPlanLogicalOptimizer.java: Loading...
  25. /trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java: Loading...
  26. /trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java: Loading...
  27. /trunk/test/org/apache/pig/test/Util.java: Loading...
  28. /trunk/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java: Loading...
/trunk/src/org/apache/pig/PigServer.java
Revision 1339793 New Change
[20] 938 lines
[+20] [+] public ExecJob store(String id, String filename, String func)
939
    throws IOException {
939
    throws IOException {
940
        currDAG.parseQuery();
940
        currDAG.parseQuery();
941
        currDAG.buildPlan( alias );
941
        currDAG.buildPlan( alias );
942

    
   
942

   
943
        try {
943
        try {
944
            QueryParserUtils.attachStorePlan( currDAG.lp, filename, func, currDAG.getOperator( alias ), alias, pigContext );
944
            QueryParserUtils.attachStorePlan(scope, currDAG.lp, filename, func, currDAG.getOperator( alias ), alias, pigContext);
945
            currDAG.compile();
945
            currDAG.compile();
946
            return executeCompiledLogicalPlan();
946
            return executeCompiledLogicalPlan();
947
        } catch (PigException e) {
947
        } catch (PigException e) {
948
            int errCode = 1002;
948
            int errCode = 1002;
949
            String msg = "Unable to store alias " + alias;
949
            String msg = "Unable to store alias " + alias;
[+20] [20] 271 lines
[+20] [+] private void buildStorePlan(String alias) throws IOException {
1221
        currDAG.buildPlan( alias );
1221
        currDAG.buildPlan( alias );
1222

    
   
1222

   
1223
        if( !isBatchOn() || alias != null ) {
1223
        if( !isBatchOn() || alias != null ) {
1224
            // MRCompiler needs a store to be the leaf - hence
1224
            // MRCompiler needs a store to be the leaf - hence
1225
            // add a store to the plan to explain
1225
            // add a store to the plan to explain
1226
            QueryParserUtils.attachStorePlan( currDAG.lp, "fakefile", null, currDAG.getOperator( alias ), 
1226
            QueryParserUtils.attachStorePlan(scope, currDAG.lp, "fakefile", null, currDAG.getOperator( alias ), 
1227
                    "fake", pigContext );
1227
                    "fake", pigContext );
1228
        }
1228
        }
1229
        currDAG.compile();
1229
        currDAG.compile();
1230
    }
1230
    }
1231

    
   
1231

   
[+20] [20] 392 lines
[+20] [+] private String buildQuery() {
1624
            compile( lp );
1624
            compile( lp );
1625
            currDAG.postProcess();
1625
            currDAG.postProcess();
1626
        }
1626
        }
1627
        
1627
        
1628
        private void compile(LogicalPlan lp) throws FrontendException  {
1628
        private void compile(LogicalPlan lp) throws FrontendException  {
1629
            new ColumnAliasConversionVisitor( lp ).visit();
1629
            new ColumnAliasConversionVisitor(lp).visit();
1630
            new SchemaAliasVisitor( lp ).visit();
1630
            new SchemaAliasVisitor(lp).visit();
1631
            new ScalarVisitor( lp, pigContext ).visit();
1631
            new ScalarVisitor(lp, pigContext, scope).visit();
1632
            
1632
            
1633
            // TODO: move optimizer here from HExecuteEngine.
1633
            // TODO: move optimizer here from HExecuteEngine.
1634
            // TODO: input/output validation visitor
1634
            // TODO: input/output validation visitor
1635

    
   
1635

   
1636
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
1636
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
[+20] [20] 117 lines
/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/rules/LoadStoreFuncDupSignatureValidator.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/parser/FunctionType.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
Revision 1339793 New Change
 
/trunk/src/org/apache/pig/parser/QueryParserUtils.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/TestLoadStoreFuncLifeCycle.java
New File
 
/trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java
New File
 
/trunk/test/org/apache/pig/parser/TestScalarVisitor.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestMRCompiler.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestNewPlanFilterRule.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestNewPlanListener.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestNewPlanLogicalOptimizer.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/Util.java
Revision 1339793 New Change
 
/trunk/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
Revision 1339793 New Change
 
  1. /trunk/src/org/apache/pig/PigServer.java: Loading...
  2. /trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java: Loading...
  3. /trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java: Loading...
  4. /trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java: Loading...
  5. /trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java: Loading...
  6. /trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java: Loading...
  7. /trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java: Loading...
  8. /trunk/src/org/apache/pig/newplan/logical/rules/LoadStoreFuncDupSignatureValidator.java: Loading...
  9. /trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java: Loading...
  10. /trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java: Loading...
  11. /trunk/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java: Loading...
  12. /trunk/src/org/apache/pig/parser/FunctionType.java: Loading...
  13. /trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java: Loading...
  14. /trunk/src/org/apache/pig/parser/QueryParserUtils.java: Loading...
  15. /trunk/test/org/apache/pig/TestLoadStoreFuncLifeCycle.java: Loading...
  16. /trunk/test/org/apache/pig/newplan/logical/relational/LOTestHelper.java: Loading...
  17. /trunk/test/org/apache/pig/parser/TestScalarVisitor.java: Loading...
  18. /trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java: Loading...
  19. /trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java: Loading...
  20. /trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java: Loading...
  21. /trunk/test/org/apache/pig/test/TestMRCompiler.java: Loading...
  22. /trunk/test/org/apache/pig/test/TestNewPlanFilterRule.java: Loading...
  23. /trunk/test/org/apache/pig/test/TestNewPlanListener.java: Loading...
  24. /trunk/test/org/apache/pig/test/TestNewPlanLogicalOptimizer.java: Loading...
  25. /trunk/test/org/apache/pig/test/TestNewPlanOperatorPlan.java: Loading...
  26. /trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java: Loading...
  27. /trunk/test/org/apache/pig/test/Util.java: Loading...
  28. /trunk/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java: Loading...