Review Board 1.7.22


SQOOP-1154 Sqoop2: Text partitioner might miss or include edge values

Review Request #13051 - Created July 30, 2013 and updated

Jarek Cecho
SQOOP-1154
Reviewers
Sqoop
sqoop-sqoop2
I've altered the methods to inject the lower bound into first partition and upper bound to the last partition.
Altered one unit test and tested in the new integration test that is part of SQOOP-1114.
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
Revision 8d0c4ab017790985741c7effc6e1e58c54267a02 New Change
[20] 247 lines
[+20] [+] public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfiguration, ImportJobConfiguration> {
248
    BigDecimal maxStringBD = textToBigDecimal(maxStringValue);
248
    BigDecimal maxStringBD = textToBigDecimal(maxStringValue);
249

    
   
249

   
250
    // Having one single value means that we can create only one single split
250
    // Having one single value means that we can create only one single split
251
    if(minStringBD.equals(maxStringBD)) {
251
    if(minStringBD.equals(maxStringBD)) {
252
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
252
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
253
      partition.setConditions(constructTextConditions(prefix, maxStringBD));
253
      partition.setConditions(constructTextConditions(prefix, 0, 0,

    
   
254
        partitionMinValue, partitionMaxValue, true, true));
254
      partitions.add(partition);
255
      partitions.add(partition);
255
      return partitions;
256
      return partitions;
256
    }
257
    }
257

    
   
258

   
258
    // Get all the split points together.
259
    // Get all the split points together.
[+20] [20] 33 lines
[+20] public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfiguration, ImportJobConfiguration> {
292
    BigDecimal start = splitPoints.get(0);
293
    BigDecimal start = splitPoints.get(0);
293
    for (int i = 1; i < splitPoints.size(); i++) {
294
    for (int i = 1; i < splitPoints.size(); i++) {
294
      BigDecimal end = splitPoints.get(i);
295
      BigDecimal end = splitPoints.get(i);
295

    
   
296

   
296
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
297
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
297
      partition.setConditions(constructTextConditions(prefix, start,
298
      partition.setConditions(constructTextConditions(prefix, start, end,
298
          end, i == splitPoints.size() - 1));
299
        partitionMinValue, partitionMaxValue, i == 1, i == splitPoints.size() - 1));
299
      partitions.add(partition);
300
      partitions.add(partition);
300

    
   
301

   
301
      start = end;
302
      start = end;
302
    }
303
    }
303

    
   
304

   
[+20] [20] 215 lines
[+20] [+] protected String constructDateConditions(SimpleDateFormat sdf,
519
    conditions.append(lastOne ? " <= " : " < ");
520
    conditions.append(lastOne ? " <= " : " < ");
520
    conditions.append('\'').append(sdf.format((java.util.Date)upperBound)).append('\'');
521
    conditions.append('\'').append(sdf.format((java.util.Date)upperBound)).append('\'');
521
    return conditions.toString();
522
    return conditions.toString();
522
  }
523
  }
523

    
   
524

   
524
  protected String constructTextConditions(String prefix,
525
  protected String constructTextConditions(String prefix, Object lowerBound, Object upperBound,
525
      Object lowerBound, Object upperBound, boolean lastOne) {
526
      String lowerStringBound, String upperStringBound, boolean firstOne, boolean lastOne) {
526
    StringBuilder conditions = new StringBuilder();
527
    StringBuilder conditions = new StringBuilder();
527
    String lbString = prefix + bigDecimalToText((BigDecimal)lowerBound);
528
    String lbString = prefix + bigDecimalToText((BigDecimal)lowerBound);
528
    String ubString = prefix + bigDecimalToText((BigDecimal)upperBound);
529
    String ubString = prefix + bigDecimalToText((BigDecimal)upperBound);
529
    conditions.append('\'').append(lbString).append('\'');
530
    conditions.append('\'').append(firstOne ? lowerStringBound : lbString).append('\'');
530
    conditions.append(" <= ");
531
    conditions.append(" <= ");
531
    conditions.append(partitionColumnName);
532
    conditions.append(partitionColumnName);
532
    conditions.append(" AND ");
533
    conditions.append(" AND ");
533
    conditions.append(partitionColumnName);
534
    conditions.append(partitionColumnName);
534
    conditions.append(lastOne ? " <= " : " < ");
535
    conditions.append(lastOne ? " <= " : " < ");
535
    conditions.append('\'').append(ubString).append('\'');
536
    conditions.append('\'').append(lastOne ? upperStringBound : ubString).append('\'');
536
    return conditions.toString();
537
    return conditions.toString();
537
  }
538
  }
538

    
   
539

   
539
  protected String constructTextConditions(String prefix, Object value) {

   
540
    return new StringBuilder()

   
541
      .append(partitionColumnName)

   
542
      .append(" = ").append('\'')

   
543
      .append(prefix + bigDecimalToText((BigDecimal)value))

   
544
      .append('\'').toString()

   
545
     ;

   
546
  }

   
547

    
   

   
548

    
   

   
549
  /**
540
  /**
550
   *  Converts a string to a BigDecimal representation in Base 2^21 format.
541
   *  Converts a string to a BigDecimal representation in Base 2^21 format.
551
   *  The maximum Unicode code point value defined is 10FFFF.  Although
542
   *  The maximum Unicode code point value defined is 10FFFF.  Although
552
   *  not all database system support UTF16 and mostly we expect UCS2
543
   *  not all database system support UTF16 and mostly we expect UCS2
553
   *  characters only, for completeness, we assume that all the unicode
544
   *  characters only, for completeness, we assume that all the unicode
[+20] [20] 61 lines
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
Revision a33dd6cfe7b001f173e155b3fb694c5eb6e7368e New Change
 
  1. connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java: Loading...
  2. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java: Loading...