Review Board 1.7.22


SQOOP-1121 Sqoop2: Serialize schema and make them available in the MR job.

Review Request #12261 - Created July 3, 2013 and submitted

Jarek Cecho
SQOOP-1121
Reviewers
Sqoop
sqoop-sqoop2
The patch seems big, but it's just touching a lot of files. Couple of high level notes:

* I've added serialization of the schema into mapreduce job
* I've refactored a bit ConfigurationUtils by providing better encapsulation and adding unit tests for that class.
* I've added the schema to all MR context objects (that are passed to extractor, loader, ...)
* Currently only Connector schema is being used everywhere, I'm expecting to change that once Framework (=hio) schema will be in place.
Provided new unit test for the util class ConfigurationUtils. Both unit and integration tests seems to be working, plus I've tested the changes on real cluster.

Diff revision 2

This is not the most recent revision of the diff. The latest diff is revision 3. See what's changed.

1 2 3
1 2 3

  1. common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java: Loading...
  2. common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java: Loading...
  3. common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java: Loading...
  4. common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java: Loading...
  5. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java: Loading...
  6. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java: Loading...
  7. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java: Loading...
  8. core/src/main/java/org/apache/sqoop/framework/JobManager.java: Loading...
  9. execution/mapreduce/pom.xml: Loading...
  10. execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java: Loading...
  11. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java: Loading...
  12. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java: Loading...
  13. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java: Loading...
  14. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java: Loading...
  15. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java: Loading...
  16. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java: Loading...
  17. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java: Loading...
  18. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java: Loading...
  19. execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java: Loading...
  20. execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java: Loading...
  21. submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java: Loading...
common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java
Revision 10cfb103b97a367c87c9ab264bf83035a5547660 New Change
[20] 15 lines
[+20]
16
 * limitations under the License.
16
 * limitations under the License.
17
 */
17
 */
18
package org.apache.sqoop.job.etl;
18
package org.apache.sqoop.job.etl;
19

    
   
19

   
20
import org.apache.sqoop.common.ImmutableContext;
20
import org.apache.sqoop.common.ImmutableContext;

    
   
21
import org.apache.sqoop.schema.Schema;
21

    
   
22

   
22
/**
23
/**
23
 * Context implementation for Destroyer.
24
 * Context implementation for Destroyer.
24
 *
25
 *
25
 * This class is wrapping information if the run was successful or not.
26
 * This class is wrapping information if the run was successful or not.
26
 */
27
 */
27
public class DestroyerContext extends ActorContext {
28
public class DestroyerContext extends ActorContext {
28

    
   
29

   
29
  private boolean success;
30
  private boolean success;
30

    
   
31

   
31
  public DestroyerContext(ImmutableContext context, boolean success) {
32
  private Schema schema;

    
   
33

   

    
   
34
  public DestroyerContext(ImmutableContext context, boolean success, Schema schema) {
32
    super(context);
35
    super(context);
33
    this.success = success;
36
    this.success = success;

    
   
37
    this.schema = schema;
34
  }
38
  }
35

    
   
39

   
36
  /**
40
  /**
37
   * Return true if the job was successful.
41
   * Return true if the job was successful.
38
   *
42
   *
39
   * @return True if the job was successful
43
   * @return True if the job was successful
40
   */
44
   */
41
  public boolean isSuccess() {
45
  public boolean isSuccess() {
42
    return success;
46
    return success;
43
  }
47
  }

    
   
48

   

    
   
49
  /**

    
   
50
   * Return schema associated with this step.

    
   
51
   *

    
   
52
   * @return

    
   
53
   */

    
   
54
  public Schema getSchema() {

    
   
55
    return schema;

    
   
56
  }
44
}
57
}
common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
Revision f9d7a8b57b999db4531e46ce62e2ac535e580e3f New Change
 
common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java
Revision dad19f15672375084d3c9d6736c19628853222c9 New Change
 
common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java
Revision 5e7cea75cb5d683c4bdc47674a7b6ecc56881b32 New Change
 
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
Revision 50a32d91f3a6f2290a6519754de9427e167df20b New Change
 
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java
Revision 54ffe5ba44a12ca71b08cd93a9ed347255cb8d25 New Change
 
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
Revision 0afec49f1b46d2b0c8b30a029507f5de6f7e7ee6 New Change
 
core/src/main/java/org/apache/sqoop/framework/JobManager.java
Revision 58d6c104bbc589f33857f43f8831448635758ce8 New Change
 
execution/mapreduce/pom.xml
Revision 31da5f11887415db85e26c6ccb2500383d9b2635 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
Revision e2b3ce84484c8a5a984a8b31a4d83e5ca03c0ba4 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
Revision 64ec4375fd4c2edb48f605b4997cccd29d420a22 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
Revision 4493a45f720c015d285030e5b7b1382e1f3f16e7 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
Revision 0721b7e20f00c0d8559356919ca79bbdcbe669b5 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
Revision 7715d5f7be23c838d337f2c117af330843e99e8f New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
Revision d47f861035a9eb199d166dc9f54542654b63a9b8 New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java
Revision 58c306824ca307eb062e6664cb1962072cb5526b New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java
Revision b9b063193d39b005a1c36bf7a5399559c39ac925 New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
Revision ee03427ba0ca7009c0ea2edf15e4f482f6584ed2 New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java
New File
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
Revision d2b501eb56ae0708754047150e0618e1f6470395 New Change
 
submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
Revision 0e8c9f72339f1f6d82ce2a7c49cc1645369f480d New Change
 
  1. common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java: Loading...
  2. common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java: Loading...
  3. common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java: Loading...
  4. common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java: Loading...
  5. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java: Loading...
  6. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java: Loading...
  7. connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java: Loading...
  8. core/src/main/java/org/apache/sqoop/framework/JobManager.java: Loading...
  9. execution/mapreduce/pom.xml: Loading...
  10. execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java: Loading...
  11. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java: Loading...
  12. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java: Loading...
  13. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java: Loading...
  14. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java: Loading...
  15. execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java: Loading...
  16. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java: Loading...
  17. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java: Loading...
  18. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java: Loading...
  19. execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java: Loading...
  20. execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java: Loading...
  21. submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java: Loading...