Review Board 1.7.22


SQOOP-737 Give user option to set number of extractors and loaders

Review Request #8339 - Created Dec. 4, 2012 and submitted

Jarek Cecho
SQOOP-737
Reviewers
Sqoop
sqoop-sqoop2
I've implemented end-to-end behaviour to cover setting number of extractors and loaders. Current default is to have 10 extractors (=mappers) and equal number of loaders as extractors (= no reduce phase). Setting any number of loaders will add reduce phase.
Unit tests are passing and I've tested it on real cluster.
common/src/main/java/org/apache/sqoop/common/MutableContext.java
Revision 238bbfd4d621bd0b091a5a1850d7926da6431524 New Change
[20] 27 lines
[+20]
28
   * @param key Key
28
   * @param key Key
29
   * @param value New value
29
   * @param value New value
30
   */
30
   */
31
  public void setString(String key, String value);
31
  public void setString(String key, String value);
32

    
   
32

   

    
   
33
  /**

    
   
34
   * Set long value for given key.

    
   
35
   *

    
   
36
   * @param key Key

    
   
37
   * @param value New value

    
   
38
   */

    
   
39
  public void setLong(String key, long value);

    
   
40

   

    
   
41
  /**

    
   
42
   * Set integer value for given key.

    
   
43
   *

    
   
44
   * @param key Key

    
   
45
   * @param value New value

    
   
46
   */

    
   
47
  public void setInteger(String key, int value);
33
}
48
}
common/src/main/java/org/apache/sqoop/common/MutableMapContext.java
Revision cd9d3e3f9a6a79cb27bf21fd605331727d60b09e New Change
 
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
Revision a6d3b52fed2fd7a3f34e787603a3feeb1a0c6fd8 New Change
 
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
Revision d5db190f831d977f4414fb0e27498885dbd03636 New Change
 
core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java
Revision 0cd696982772e9dd80e7584435cd5be0106bfd97 New Change
 
core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
Revision fb6b6a94722baf5c29292d72ed8908cc1e2140d7 New Change
 
core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java
Revision 8c4dcf168a8df70b727d58e108d3df987e643bf3 New Change
 
core/src/main/java/org/apache/sqoop/framework/configuration/ThrottlingForm.java
New File
 
core/src/main/resources/framework-resources.properties
Revision 019f5ca8581e466e8187549ccaa7e3f2efd74dbf New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
Revision 3248e7713388f990c47261cf77d65ce46444a0b4 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
Revision d899fcef662ed69b567762e55c40840c66fc2387 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/MapreduceExecutionError.java
Revision 30956f30cb4eb29b5c2a917c80be99a6696677e8 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
Revision 8fcdc99daa08a02775ed2ba0708eccfdfdbe078b New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java
Revision 21a2be9d7ca4fa5d3ff9dc95ad7a50c622560691 New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
Revision 745a3a42bff9d74a2d62438ff0cbcc8babb00e26 New Change
 
spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
Revision 3a525c4fed5fdc8372f83b72c5a37538955618b8 New Change
 
submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
Revision a64a477577b84afb0d0848516186682dda9b6310 New Change
 
  1. common/src/main/java/org/apache/sqoop/common/MutableContext.java: Loading...
  2. common/src/main/java/org/apache/sqoop/common/MutableMapContext.java: Loading...
  3. connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java: Loading...
  4. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java: Loading...
  5. core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java: Loading...
  6. core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java: Loading...
  7. core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java: Loading...
  8. core/src/main/java/org/apache/sqoop/framework/configuration/ThrottlingForm.java: Loading...
  9. core/src/main/resources/framework-resources.properties: Loading...
  10. execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java: Loading...
  11. execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java: Loading...
  12. execution/mapreduce/src/main/java/org/apache/sqoop/job/MapreduceExecutionError.java: Loading...
  13. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java: Loading...
  14. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java: Loading...
  15. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java: Loading...
  16. spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java: Loading...
  17. submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java: Loading...