Review Board 1.7.22


[PIG-3173] - Partition filter pushdown does not happen if partition keys condition include a AND and OR construct

Review Request #10035 - Created March 19, 2013 and submitted

Rohini Palaniswamy
PIG-3173
Reviewers
pig
pig
1) Fixed cases where partition pushdown was not happening for AND and OR construct
2) Commented out the negative test cases as they were actually not asserting anything.
Unit tests added and tested few cases manually with hcat.

Diff revision 1

This is not the most recent revision of the diff. The latest diff is revision 2. See what's changed.

1 2
1 2

  1. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/PColFilterExtractor.java: Loading...
  2. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java: Loading...
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/PColFilterExtractor.java
Revision 1458047 New Change
[20] 176 lines
[+20] [+] protected void visit(ProjectExpression project) throws FrontendException {
177
                + fieldName + ") inside a bincond operator in the " +
177
                + fieldName + ") inside a bincond operator in the " +
178
                "filter condition.");
178
                "filter condition.");
179
            canPushDown = false;
179
            canPushDown = false;
180
            return;
180
            return;
181
			}
181
			}
182
			opsToCheckFor.set(0, AndExpression.class);

   
183
			opsToCheckFor.add(OrExpression.class);

   
184
			if(checkSuccessors(project, opsToCheckFor)) {

   
185
            LOG.warn("No partition filter push down: " +

   
186
                "You have an partition column (" + fieldName +

   
187
                " ) in a construction like: " +

   
188
                "(pcond  and ...) or (pcond and ...) " +

   
189
                "where pcond is a condition on a partition column.");

   
190
            canPushDown = false;

   
191
            return;
Moved to 214

   
192
			}
Moved to 215

   
193
		} else {
182
		} else {
194
			sawNonKeyCol = true;
183
			sawNonKeyCol = true;
195
		}
184
		}
196
	}
185
	}
197

    
   
186

   
198
	private void visit(BinaryExpression binOp) throws FrontendException {
187
	private void visit(BinaryExpression binOp) throws FrontendException {
199
		boolean lhsSawKey = false;
188
		boolean lhsSawKey = false;
200
		boolean rhsSawKey = false;
189
		boolean rhsSawKey = false;
201
		boolean lhsSawNonKeyCol = false;
190
		boolean lhsSawNonKeyCol = false;
202
		boolean rhsSawNonKeyCol = false;
191
		boolean rhsSawNonKeyCol = false;
203

    
   

   
204
		sawKey = false;
192
        sawKey = false;
205
		sawNonKeyCol = false;
193
        sawNonKeyCol = false;

    
   
194

   

    
   
195
        if (binOp instanceof OrExpression && binOp.getLhs() instanceof AndExpression &&

    
   
196
                binOp.getRhs() instanceof AndExpression) {

    
   
197
            visit(binOp.getLhs());

    
   
198
            lhsSawNonKeyCol = sawNonKeyCol;

    
   
199
            this.replaceSide = Side.NONE;

    
   
200
            visit(binOp.getRhs());

    
   
201
            this.replaceSide = Side.NONE;

    
   
202
            rhsSawNonKeyCol = sawNonKeyCol;

    
   
203
            if (lhsSawNonKeyCol || rhsSawNonKeyCol || !canPushDown) {

    
   
204
                sawKey = false;

    
   
205
                sawNonKeyCol = true;

    
   
206
                // Don't set canPushDown to false. If there are other AND

    
   
207
                // conditions on a partition column we want to push that down

    
   
208
                // canPushDown = false;

    
   
209
                LOG.warn("No partition filter push down: You have partition and non-partition "

    
   
210
                        + "columns  in a construction like: "

    
   
211
                        + "(pcond and non-pcond ..) or (pcond and non-pcond ...) "

    
   
212
                        + "where pcond is a condition on a partition column and "

    
   
213
                        + "non-pcond is a condition on a non-partition column.");
Moved from 191

    
   
214
                return;
Moved from 192

    
   
215
            }

    
   
216
        }
206
		visit( binOp.getLhs() );
217
		visit( binOp.getLhs() );
207
		replaceChild(binOp.getLhs());
218
		replaceChild(binOp.getLhs());
208
		lhsSawKey = sawKey;
219
		lhsSawKey = sawKey;
209
		lhsSawNonKeyCol = sawNonKeyCol;
220
		lhsSawNonKeyCol = sawNonKeyCol;
210

    
   
221

   
211
		sawKey = false;
222
		sawKey = false;
212
		sawNonKeyCol = false;
223
		sawNonKeyCol = false;
213
		visit( binOp.getRhs() );
224
		visit( binOp.getRhs() );
214
		replaceChild(binOp.getRhs());
225
		replaceChild(binOp.getRhs());
215
		rhsSawKey = sawKey;
226
		rhsSawKey = sawKey;
216
		rhsSawNonKeyCol = sawNonKeyCol;
227
		rhsSawNonKeyCol = sawNonKeyCol;
217

    
   
228
		
218
		// only in the case of an AND, we potentially split the AND to
229
		// only in the case of an AND, we potentially split the AND to
219
		// remove conditions on partition columns out of the AND. For this
230
		// remove conditions on partition columns out of the AND. For this
220
		// we set replaceSide accordingly so that when we reach a predecessor
231
		// we set replaceSide accordingly so that when we reach a predecessor
221
		// we can trim the appropriate side. If both sides of the AND have
232
		// we can trim the appropriate side. If both sides of the AND have
222
		// conditions on partition columns, we will remove the AND completely -
233
		// conditions on partition columns, we will remove the AND completely -
[+20] [20] 324 lines
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
Revision 1458047 New Change
 
  1. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/PColFilterExtractor.java: Loading...
  2. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java: Loading...