Review Board 1.7.22


[PIG-3173] - Partition filter pushdown does not happen if partition keys condition include a AND and OR construct

Review Request #10035 - Created March 19, 2013 and submitted

Rohini Palaniswamy
PIG-3173
Reviewers
pig
pig
1) Fixed cases where partition pushdown was not happening for AND and OR construct
2) Commented out the negative test cases as they were actually not asserting anything.
Unit tests added and tested few cases manually with hcat.
http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/PColFilterExtractor.java
Revision 1467858 New Change
[20] 176 lines
[+20] [+] protected void visit(ProjectExpression project) throws FrontendException {
177
                + fieldName + ") inside a bincond operator in the " +
177
                + fieldName + ") inside a bincond operator in the " +
178
                "filter condition.");
178
                "filter condition.");
179
            canPushDown = false;
179
            canPushDown = false;
180
            return;
180
            return;
181
			}
181
			}
182
			opsToCheckFor.set(0, AndExpression.class);

   
183
			opsToCheckFor.add(OrExpression.class);

   
184
			if(checkSuccessors(project, opsToCheckFor)) {

   
185
            LOG.warn("No partition filter push down: " +

   
186
                "You have an partition column (" + fieldName +

   
187
                " ) in a construction like: " +

   
188
                "(pcond  and ...) or (pcond and ...) " +

   
189
                "where pcond is a condition on a partition column.");

   
190
            canPushDown = false;

   
191
            return;
Moved to 218

   
192
			}
Moved to 219

   
193
		} else {
182
		} else {
194
			sawNonKeyCol = true;
183
			sawNonKeyCol = true;
195
		}
184
		}
196
	}
185
	}
197

    
   
186

   
198
	private void visit(BinaryExpression binOp) throws FrontendException {
187
	private void visit(BinaryExpression binOp) throws FrontendException {
199
		boolean lhsSawKey = false;
188
		boolean lhsSawKey = false;
200
		boolean rhsSawKey = false;
189
		boolean rhsSawKey = false;
201
		boolean lhsSawNonKeyCol = false;
190
		boolean lhsSawNonKeyCol = false;
202
		boolean rhsSawNonKeyCol = false;
191
		boolean rhsSawNonKeyCol = false;
203

    
   

   
204
		sawKey = false;
192
        sawKey = false;
205
		sawNonKeyCol = false;
193
        sawNonKeyCol = false;

    
   
194

   

    
   
195
        LogicalExpression binLHS = binOp.getLhs();

    
   
196
        LogicalExpression binRHS = binOp.getRhs();

    
   
197
        // Take care of nested OR as in

    
   
198
        // ((cond1 and cond2) or (cond3 and cond4) or (cond5 and cond6)) or (cond7 and cond8)

    
   
199
        if (binOp instanceof OrExpression &&

    
   
200
                ((binLHS instanceof AndExpression && binRHS instanceof AndExpression) ||

    
   
201
                  binLHS instanceof OrExpression || binRHS instanceof OrExpression)) {

    
   
202
            visit(binLHS);

    
   
203
            lhsSawNonKeyCol = sawNonKeyCol;

    
   
204
            this.replaceSide = Side.NONE;

    
   
205
            visit(binRHS);

    
   
206
            rhsSawNonKeyCol = sawNonKeyCol;

    
   
207
            this.replaceSide = Side.NONE;

    
   
208
            if (lhsSawNonKeyCol || rhsSawNonKeyCol || !canPushDown) {

    
   
209
                sawKey = false;

    
   
210
                sawNonKeyCol = true;

    
   
211
                // Don't set canPushDown to false. If there are other AND

    
   
212
                // conditions on a partition column we want to push that down

    
   
213
                LOG.warn("No partition filter push down: You have partition and non-partition "

    
   
214
                        + "columns  in a construction like: "

    
   
215
                        + "(pcond and non-pcond ..) or (pcond and non-pcond ...) "

    
   
216
                        + "where pcond is a condition on a partition column and "

    
   
217
                        + "non-pcond is a condition on a non-partition column.");
Moved from 191

    
   
218
                return;
Moved from 192

    
   
219
            }

    
   
220
        }
206
		visit( binOp.getLhs() );
221
		visit( binOp.getLhs() );
207
		replaceChild(binOp.getLhs());
222
		replaceChild(binOp.getLhs());
208
		lhsSawKey = sawKey;
223
		lhsSawKey = sawKey;
209
		lhsSawNonKeyCol = sawNonKeyCol;
224
		lhsSawNonKeyCol = sawNonKeyCol;
210

    
   
225

   
[+20] [20] 27 lines
[+20] private void visit(BinaryExpression binOp) throws FrontendException {
238

    
   
253

   
239
		sawKey = lhsSawKey || rhsSawKey;
254
		sawKey = lhsSawKey || rhsSawKey;
240
		sawNonKeyCol = lhsSawNonKeyCol || rhsSawNonKeyCol;
255
		sawNonKeyCol = lhsSawNonKeyCol || rhsSawNonKeyCol;
241
	}
256
	}
242

    
   
257

   
243

    
   

   
244

    
   

   
245
	/**
258
	/**
246
	 * @return the condition on partition columns extracted from filter
259
	 * @return the condition on partition columns extracted from filter
247
	 */
260
	 */
248
	public  Expression getPColCondition(){
261
	public  Expression getPColCondition(){
249
    if(!canPushDown || pColConditions.size() == 0)
262
    if(!canPushDown || pColConditions.size() == 0)
[+20] [20] 297 lines
http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
Revision 1467858 New Change
 
  1. http://svn.apache.org/repos/asf/pig/trunk/src/org/apache/pig/newplan/PColFilterExtractor.java: Loading...
  2. http://svn.apache.org/repos/asf/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java: Loading...