Review Board 1.7.22


Fix for SQOOP-1113

Review Request #12255 - Created July 3, 2013 and updated

Venkat Ranganathan
SQOOP-1113
Reviewers
Sqoop
sqoop-sqoop2
Fix for text partitioning issue.  I also added a new test to test this case. 
Added a new test.   All tests pass
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
Revision 0bb979e New Change
[20] 242 lines
[+20] [+] public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfiguration, ImportJobConfiguration> {
243
      splitSize = NUMERIC_MIN_INCREMENT;
243
      splitSize = NUMERIC_MIN_INCREMENT;
244
    }
244
    }
245

    
   
245

   
246
    BigDecimal curVal = minStringBD;
246
    BigDecimal curVal = minStringBD;
247

    
   
247

   
248
    while (curVal.compareTo(maxStringBD) <= 0) {
248
    int parts = 0;

    
   
249

   

    
   
250
    while (curVal.compareTo(maxStringBD) <= 0 && parts < numberPartitions) {
249
      splitPoints.add(curVal);
251
      splitPoints.add(curVal);
250
      curVal = curVal.add(splitSize);
252
      curVal = curVal.add(splitSize);

    
   
253
      // bigDecimalToText approximates to next comparison location.

    
   
254
      // Make sure we are still in range

    
   
255
      String text = bigDecimalToText(curVal);

    
   
256
      curVal = textToBigDecimal(text);

    
   
257
      ++parts;
251
    }
258
    }
252

    
   
259

   
253
    if (splitPoints.size() == 0
260
    if (splitPoints.size() == 0
254
        || splitPoints.get(0).compareTo(minStringBD) != 0) {
261
        || splitPoints.get(0).compareTo(minStringBD) != 0) {
255
      splitPoints.add(0, minStringBD);
262
      splitPoints.add(0, minStringBD);
[+20] [20] 273 lines
[+20] [+] protected String constructTextConditions(String prefix, Object value) {
529
   *  This can be split and each split point can be converted back to
536
   *  This can be split and each split point can be converted back to
530
   *  a string value for comparison purposes.   The number of characters
537
   *  a string value for comparison purposes.   The number of characters
531
   *  is restricted to prevent repeating fractions and rounding errors
538
   *  is restricted to prevent repeating fractions and rounding errors
532
   *  towards the higher fraction positions.
539
   *  towards the higher fraction positions.
533
   */
540
   */
534
  private static final BigDecimal UNITS_BASE = new BigDecimal(2097152);
541
  private static final BigDecimal UNITS_BASE = new BigDecimal(0x200000);
535
  private static final int MAX_CHARS_TO_CONVERT = 4;
542
  private static final int MAX_CHARS_TO_CONVERT = 4;
536

    
   
543

   
537
  private BigDecimal textToBigDecimal(String str) {
544
  private BigDecimal textToBigDecimal(String str) {
538
    BigDecimal result = BigDecimal.ZERO;
545
    BigDecimal result = BigDecimal.ZERO;
539
    BigDecimal divisor = UNITS_BASE;
546
    BigDecimal divisor = UNITS_BASE;
[+20] [20] 16 lines
[+20] [+] private String bigDecimalToText(BigDecimal bd) {
556
    StringBuilder sb = new StringBuilder();
563
    StringBuilder sb = new StringBuilder();
557

    
   
564

   
558
    for (int n = 0; n < MAX_CHARS_TO_CONVERT; ++n) {
565
    for (int n = 0; n < MAX_CHARS_TO_CONVERT; ++n) {
559
      curVal = curVal.multiply(UNITS_BASE);
566
      curVal = curVal.multiply(UNITS_BASE);
560
      int cp = curVal.intValue();
567
      int cp = curVal.intValue();
561
      if (0 == cp) {
568
      if (0 >= cp) {

    
   
569
        break;

    
   
570
      }

    
   
571

   

    
   
572
      if (!Character.isDefined(cp)) {

    
   
573
        int t_cp = Character.MAX_CODE_POINT < cp ? 1 : cp;

    
   
574
        // We are guaranteed to find at least one character

    
   
575
        while(!Character.isDefined(t_cp)) {

    
   
576
          ++t_cp;

    
   
577
          if (t_cp == cp) {
562
        break;
578
            break;
563
      }
579
          }

    
   
580
          if (t_cp >= Character.MAX_CODE_POINT || t_cp <= 0)  {

    
   
581
            t_cp = 1;

    
   
582
          }

    
   
583
        }

    
   
584
        cp = t_cp;

    
   
585
      }
564
      curVal = curVal.subtract(new BigDecimal(cp));
586
      curVal = curVal.subtract(new BigDecimal(cp));
565
      sb.append(Character.toChars(cp));
587
      sb.append(Character.toChars(cp));
566
    }
588
    }

    
   
589

   
567
    return sb.toString();
590
    return sb.toString();
568
  }
591
  }
569

    
   
592

   
570
}
593
}
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
Revision 1a3601a New Change
 
  1. connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java: Loading...
  2. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java: Loading...