Review Board 1.7.22


PIG-2167 - Naive implementation of CUBE operator

Review Request #4670 - Created April 6, 2012 and updated

Prasanth_J
PIG-2167
Reviewers
pig
dvryaboy
pig
This is a review board for https://issues.apache.org/jira/browse/PIG-2167
Unit tests: All passed

Pre-commit tests: All passed
ant clean test-commit

Changes between revision 1 and 2

1 2 3
1 2 3

  1. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/AllExpressionVisitor.java: Loading...
  2. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java: Loading...
  3. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java: Loading...
  4. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java: Loading...
  5. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ProjectStarExpander.java: Loading...
  6. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AliasMasker.g: Loading...
  7. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AstPrinter.g: Loading...
  8. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AstValidator.g: Loading...
  9. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java: Loading...
  10. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g: Loading...
  11. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/QueryLexer.g: Loading...
  12. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/QueryParser.g: Loading...
  13. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestLexer.pig: Loading...
  14. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestLogicalPlanGenerator.java: Loading...
  15. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestParser.pig: Loading...
  16. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java: Loading...
  17. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestQueryParser.java: Loading...
  18. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestCubeOperator.java: Loading...
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/AllExpressionVisitor.java
Diff Revision 1 Diff Revision 2
This file contains only whitespace changes.
1
/*
1
/*
2
 * Licensed to the Apache Software Foundation (ASF) under one
2
 * Licensed to the Apache Software Foundation (ASF) under one
3
 * or more contributor license agreements.  See the NOTICE file
3
 * or more contributor license agreements.  See the NOTICE file
4
 * distributed with this work for additional information
4
 * distributed with this work for additional information
5
 * regarding copyright ownership.  The ASF licenses this file
5
 * regarding copyright ownership.  The ASF licenses this file
6
 * to you under the Apache License, Version 2.0 (the
6
 * to you under the Apache License, Version 2.0 (the
7
 * "License"); you may not use this file except in compliance
7
 * "License"); you may not use this file except in compliance
8
 * with the License.  You may obtain a copy of the License at
8
 * with the License.  You may obtain a copy of the License at
9
 *
9
 *
10
 *     http://www.apache.org/licenses/LICENSE-2.0
10
 *     http://www.apache.org/licenses/LICENSE-2.0
11
 *
11
 *
12
 * Unless required by applicable law or agreed to in writing, software
12
 * Unless required by applicable law or agreed to in writing, software
13
 * distributed under the License is distributed on an "AS IS" BASIS,
13
 * distributed under the License is distributed on an "AS IS" BASIS,
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
 * See the License for the specific language governing permissions and
15
 * See the License for the specific language governing permissions and
16
 * limitations under the License.
16
 * limitations under the License.
17
 */
17
 */
18

    
   
18

   
19
package org.apache.pig.newplan.logical.optimizer;
19
package org.apache.pig.newplan.logical.optimizer;
20

    
   
20

   
21
import java.util.Collection;
21
import java.util.Collection;
22

    
   
22

   
23
import org.apache.pig.impl.logicalLayer.FrontendException;
23
import org.apache.pig.impl.logicalLayer.FrontendException;
24
import org.apache.pig.impl.util.MultiMap;
24
import org.apache.pig.impl.util.MultiMap;
25
import org.apache.pig.newplan.OperatorPlan;
25
import org.apache.pig.newplan.OperatorPlan;
26
import org.apache.pig.newplan.PlanWalker;
26
import org.apache.pig.newplan.PlanWalker;
27
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
27
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
28
import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
28
import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
29
import org.apache.pig.newplan.logical.relational.LOCogroup;
29
import org.apache.pig.newplan.logical.relational.LOCogroup;
30
import org.apache.pig.newplan.logical.relational.LOCube;
30
import org.apache.pig.newplan.logical.relational.LOCube;
31
import org.apache.pig.newplan.logical.relational.LOFilter;
31
import org.apache.pig.newplan.logical.relational.LOFilter;
32
import org.apache.pig.newplan.logical.relational.LOForEach;
32
import org.apache.pig.newplan.logical.relational.LOForEach;
33
import org.apache.pig.newplan.logical.relational.LOGenerate;
33
import org.apache.pig.newplan.logical.relational.LOGenerate;
34
import org.apache.pig.newplan.logical.relational.LOInnerLoad;
34
import org.apache.pig.newplan.logical.relational.LOInnerLoad;
35
import org.apache.pig.newplan.logical.relational.LOJoin;
35
import org.apache.pig.newplan.logical.relational.LOJoin;
36
import org.apache.pig.newplan.logical.relational.LOLimit;
36
import org.apache.pig.newplan.logical.relational.LOLimit;
37
import org.apache.pig.newplan.logical.relational.LOSort;
37
import org.apache.pig.newplan.logical.relational.LOSort;
38
import org.apache.pig.newplan.logical.relational.LOSplitOutput;
38
import org.apache.pig.newplan.logical.relational.LOSplitOutput;
39
import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
39
import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
40
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
40
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
41

    
   
41

   
42
/**
42
/**
43
 * A visitor that walks a logical plan and then applies a given
43
 * A visitor that walks a logical plan and then applies a given
44
 * LogicalExpressionVisitor to all expressions it encounters.
44
 * LogicalExpressionVisitor to all expressions it encounters.
45
 *
45
 *
46
 */
46
 */
47
public abstract class AllExpressionVisitor extends LogicalRelationalNodesVisitor {
47
public abstract class AllExpressionVisitor extends LogicalRelationalNodesVisitor {
48
    
48
    
49
    protected LogicalRelationalOperator currentOp;
49
    protected LogicalRelationalOperator currentOp;
50

    
   
50

   
51
    /**
51
    /**
52
     * @param plan LogicalPlan to visit
52
     * @param plan LogicalPlan to visit
53
     * @param walker Walker to use to visit the plan.
53
     * @param walker Walker to use to visit the plan.
54
     */
54
     */
55
    public AllExpressionVisitor(OperatorPlan plan,
55
    public AllExpressionVisitor(OperatorPlan plan,
56
                                PlanWalker walker) throws FrontendException {
56
                                PlanWalker walker) throws FrontendException {
57
        super(plan, walker);
57
        super(plan, walker);
58
    }
58
    }
59
    
59
    
60
    /**
60
    /**
61
     * Get a new instance of the expression visitor to apply to 
61
     * Get a new instance of the expression visitor to apply to 
62
     * a given expression.
62
     * a given expression.
63
     * @param expr LogicalExpressionPlan that will be visited
63
     * @param expr LogicalExpressionPlan that will be visited
64
     * @return a new LogicalExpressionVisitor for that expression
64
     * @return a new LogicalExpressionVisitor for that expression
65
     */
65
     */
66
    abstract protected LogicalExpressionVisitor getVisitor(LogicalExpressionPlan expr) throws FrontendException;
66
    abstract protected LogicalExpressionVisitor getVisitor(LogicalExpressionPlan expr) throws FrontendException;
67
    
67
    
68
    @Override
68
    @Override
69
    public void visit(LOFilter filter) throws FrontendException {
69
    public void visit(LOFilter filter) throws FrontendException {
70
        currentOp = filter;
70
        currentOp = filter;
71
        LogicalExpressionVisitor v = getVisitor(filter.getFilterPlan());
71
        LogicalExpressionVisitor v = getVisitor(filter.getFilterPlan());
72
        v.visit();
72
        v.visit();
73
    }
73
    }
74
    
74
    
75
    @Override
75
    @Override
76
    public void visit(LOLimit limit) throws FrontendException {
76
    public void visit(LOLimit limit) throws FrontendException {
77
        currentOp = limit;
77
        currentOp = limit;
78
        if (limit.getLimitPlan() != null) {
78
        if (limit.getLimitPlan() != null) {
79
            LogicalExpressionVisitor v = getVisitor(limit.getLimitPlan());
79
            LogicalExpressionVisitor v = getVisitor(limit.getLimitPlan());
80
            v.visit();
80
            v.visit();
81
        }
81
        }
82
    }
82
    }
83
 
83
 
84
    @Override
84
    @Override
85
    public void visit(LOJoin join) throws FrontendException {
85
    public void visit(LOJoin join) throws FrontendException {
86
        currentOp = join;
86
        currentOp = join;
87
        Collection<LogicalExpressionPlan> c = join.getExpressionPlanValues();
87
        Collection<LogicalExpressionPlan> c = join.getExpressionPlanValues();
88
        for (LogicalExpressionPlan plan : c) {
88
        for (LogicalExpressionPlan plan : c) {
89
            LogicalExpressionVisitor v = getVisitor(plan);
89
            LogicalExpressionVisitor v = getVisitor(plan);
90
            v.visit();
90
            v.visit();
91
        }
91
        }
92
    }
92
    }
93
    
93
    
94
    @Override
94
    @Override
95
    public void visit(LOCube cu) throws FrontendException {
95
    public void visit(LOCube cu) throws FrontendException {
96
        currentOp = cu;
96
	currentOp = cu;
97
        MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cu.getExpressionPlans();
97
	MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cu.getExpressionPlans();
98
        for( Integer key : expressionPlans.keySet() ) {
98
	for (Integer key : expressionPlans.keySet()) {
99
            Collection<LogicalExpressionPlan> exprPlans = expressionPlans.get(key);
99
	    Collection<LogicalExpressionPlan> exprPlans = expressionPlans.get(key);
100
            for( LogicalExpressionPlan plan : exprPlans ) {
100
	    for (LogicalExpressionPlan plan : exprPlans) {
101
                LogicalExpressionVisitor v = getVisitor(plan);
101
		LogicalExpressionVisitor v = getVisitor(plan);
102
                v.visit();
102
		v.visit();
103
            }
103
	    }
104
        }
104
	}
105
    }
105
    }
106
    
106
    
107
    @Override
107
    @Override
108
    public void visit(LOCogroup cg) throws FrontendException {
108
    public void visit(LOCogroup cg) throws FrontendException {
109
        currentOp = cg;
109
        currentOp = cg;
110
        MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cg.getExpressionPlans();
110
        MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cg.getExpressionPlans();
111
        for( Integer key : expressionPlans.keySet() ) {
111
        for( Integer key : expressionPlans.keySet() ) {
112
            Collection<LogicalExpressionPlan> exprPlans = expressionPlans.get(key);
112
            Collection<LogicalExpressionPlan> exprPlans = expressionPlans.get(key);
113
            for( LogicalExpressionPlan plan : exprPlans ) {
113
            for( LogicalExpressionPlan plan : exprPlans ) {
114
                LogicalExpressionVisitor v = getVisitor(plan);
114
                LogicalExpressionVisitor v = getVisitor(plan);
115
                v.visit();
115
                v.visit();
116
            }
116
            }
117
        }
117
        }
118
    }
118
    }
119
    
119
    
120
    @Override
120
    @Override
121
    public void visit(LOForEach foreach) throws FrontendException {
121
    public void visit(LOForEach foreach) throws FrontendException {
122
        currentOp = foreach;
122
        currentOp = foreach;
123
        // We have an Inner OperatorPlan in ForEach, so we go ahead
123
        // We have an Inner OperatorPlan in ForEach, so we go ahead
124
        // and work on that plan
124
        // and work on that plan
125
        OperatorPlan innerPlan = foreach.getInnerPlan();
125
        OperatorPlan innerPlan = foreach.getInnerPlan();
126
        PlanWalker newWalker = currentWalker.spawnChildWalker(innerPlan);
126
        PlanWalker newWalker = currentWalker.spawnChildWalker(innerPlan);
127
        pushWalker(newWalker);
127
        pushWalker(newWalker);
128
        currentWalker.walk(this);
128
        currentWalker.walk(this);
129
        popWalker();
129
        popWalker();
130
    }
130
    }
131
    
131
    
132
    @Override
132
    @Override
133
    public void visit(LOGenerate gen ) throws FrontendException {
133
    public void visit(LOGenerate gen ) throws FrontendException {
134
        currentOp = gen;
134
        currentOp = gen;
135
        Collection<LogicalExpressionPlan> plans = gen.getOutputPlans();
135
        Collection<LogicalExpressionPlan> plans = gen.getOutputPlans();
136
        for( LogicalExpressionPlan plan : plans ) {
136
        for( LogicalExpressionPlan plan : plans ) {
137
            LogicalExpressionVisitor v = getVisitor(plan);
137
            LogicalExpressionVisitor v = getVisitor(plan);
138
            v.visit();
138
            v.visit();
139
        }
139
        }
140
    }
140
    }
141
    
141
    
142
    @Override
142
    @Override
143
    public void visit(LOInnerLoad load) throws FrontendException {
143
    public void visit(LOInnerLoad load) throws FrontendException {
144
        // the expression in LOInnerLoad contains info relative from LOForEach
144
        // the expression in LOInnerLoad contains info relative from LOForEach
145
        // so use LOForeach as currentOp
145
        // so use LOForeach as currentOp
146
        currentOp = load.getLOForEach();
146
        currentOp = load.getLOForEach();
147
        LogicalExpressionPlan exp = (LogicalExpressionPlan)load.getProjection().getPlan();
147
        LogicalExpressionPlan exp = (LogicalExpressionPlan)load.getProjection().getPlan();
148
       
148
       
149
        LogicalExpressionVisitor v = getVisitor(exp);
149
        LogicalExpressionVisitor v = getVisitor(exp);
150
        v.visit();       
150
        v.visit();       
151
    }
151
    }
152
    
152
    
153
    @Override
153
    @Override
154
    public void visit(LOSplitOutput splitOutput) throws FrontendException {
154
    public void visit(LOSplitOutput splitOutput) throws FrontendException {
155
        currentOp = splitOutput;
155
        currentOp = splitOutput;
156
        LogicalExpressionVisitor v = getVisitor(splitOutput.getFilterPlan());
156
        LogicalExpressionVisitor v = getVisitor(splitOutput.getFilterPlan());
157
        v.visit();
157
        v.visit();
158
    }
158
    }
159
    
159
    
160
    @Override
160
    @Override
161
    public void visit(LOSort sort) throws FrontendException {
161
    public void visit(LOSort sort) throws FrontendException {
162
        currentOp = sort;
162
        currentOp = sort;
163
        Collection<LogicalExpressionPlan> c = sort.getSortColPlans();
163
        Collection<LogicalExpressionPlan> c = sort.getSortColPlans();
164
        for (LogicalExpressionPlan plan : c) {
164
        for (LogicalExpressionPlan plan : c) {
165
            LogicalExpressionVisitor v = getVisitor(plan);
165
            LogicalExpressionVisitor v = getVisitor(plan);
166
            v.visit();
166
            v.visit();
167
        }
167
        }
168
    }
168
    }
169
}
169
}
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ProjectStarExpander.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AliasMasker.g
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AstPrinter.g
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AstValidator.g
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/QueryLexer.g
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/QueryParser.g
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestLexer.pig
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestLogicalPlanGenerator.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestParser.pig
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestQueryParser.java
Diff Revision 1 Diff Revision 2
 
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestCubeOperator.java
Diff Revision 1 Diff Revision 2
 
  1. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/AllExpressionVisitor.java: Loading...
  2. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java: Loading...
  3. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java: Loading...
  4. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java: Loading...
  5. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ProjectStarExpander.java: Loading...
  6. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AliasMasker.g: Loading...
  7. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AstPrinter.g: Loading...
  8. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/AstValidator.g: Loading...
  9. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java: Loading...
  10. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g: Loading...
  11. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/QueryLexer.g: Loading...
  12. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/parser/QueryParser.g: Loading...
  13. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestLexer.pig: Loading...
  14. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestLogicalPlanGenerator.java: Loading...
  15. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestParser.pig: Loading...
  16. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java: Loading...
  17. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/parser/TestQueryParser.java: Loading...
  18. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestCubeOperator.java: Loading...