Review Board 1.7.22


SQOOP-1046 Sqoop2: Importing single row using decimal splitter will not import anything

Review Request #11171 - Created May 15, 2013 and submitted

Jarek Cecho
SQOOP-1046
Reviewers
Sqoop
sqoop-sqoop2
I've added condition that is checking for this edge case and is creating single split.
I've added new unit tests, testing this edge case. Both unit and integration tests seems to be passing.
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
Revision 6d1a9fd71c9dac587d8bc99d7ffc64e4a57672e7 New Change
[20] 166 lines
[+20] [+] public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfiguration, ImportJobConfiguration> {
167
  }
167
  }
168

    
   
168

   
169
  protected List<Partition> partitionNumericColumn() {
169
  protected List<Partition> partitionNumericColumn() {
170
    List<Partition> partitions = new LinkedList<Partition>();
170
    List<Partition> partitions = new LinkedList<Partition>();
171

    
   
171

   
172
    // All null valeus will result in single partition
172
    // All null values will result in single partition
173
    if (partitionMinValue == null && partitionMaxValue == null) {
173
    if (partitionMinValue == null && partitionMaxValue == null) {
174
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
174
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
175
      partition.setConditions(partitionColumnName + "IS NULL");
175
      partition.setConditions(partitionColumnName + "IS NULL");
176
      partitions.add(partition);
176
      partitions.add(partition);
177
      return partitions;
177
      return partitions;
[+20] [20] 5 lines
[+20] public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfiguration, ImportJobConfiguration> {
183
    }
183
    }
184

    
   
184

   
185
    BigDecimal minValue = new BigDecimal(partitionMinValue);
185
    BigDecimal minValue = new BigDecimal(partitionMinValue);
186
    BigDecimal maxValue = new BigDecimal(partitionMaxValue);
186
    BigDecimal maxValue = new BigDecimal(partitionMaxValue);
187

    
   
187

   

    
   
188
    // Having one single value means that we can create only one single split

    
   
189
    if(minValue.equals(maxValue)) {

    
   
190
      GenericJdbcImportPartition partition = new GenericJdbcImportPartition();

    
   
191
      partition.setConditions(constructConditions(minValue));

    
   
192
      partitions.add(partition);

    
   
193
    }

    
   
194

   
188
    // Get all the split points together.
195
    // Get all the split points together.
189
    List<BigDecimal> splitPoints = new LinkedList<BigDecimal>();
196
    List<BigDecimal> splitPoints = new LinkedList<BigDecimal>();
190

    
   
197

   
191
    BigDecimal splitSize = divide(maxValue.subtract(minValue), new BigDecimal(numberPartitions));
198
    BigDecimal splitSize = divide(maxValue.subtract(minValue), new BigDecimal(numberPartitions));
192
    if (splitSize.compareTo(NUMERIC_MIN_INCREMENT) < 0) {
199
    if (splitSize.compareTo(NUMERIC_MIN_INCREMENT) < 0) {
[+20] [20] 45 lines
[+20] [+] protected String constructConditions(
238
    conditions.append(partitionColumnName);
245
    conditions.append(partitionColumnName);
239
    conditions.append(lastOne ? " <= " : " < ");
246
    conditions.append(lastOne ? " <= " : " < ");
240
    conditions.append(upperBound);
247
    conditions.append(upperBound);
241
    return conditions.toString();
248
    return conditions.toString();
242
  }
249
  }

    
   
250

   

    
   
251
  protected String constructConditions(Object value) {

    
   
252
    return new StringBuilder()

    
   
253
      .append(partitionColumnName)

    
   
254
      .append(" = ")

    
   
255
      .append(value)

    
   
256
      .toString()

    
   
257
     ;

    
   
258
  }
243
}
259
}
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
Revision 3150e7ce0490e16cba985361bd1365043aed04cc New Change
 
  1. connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java: Loading...
  2. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java: Loading...