Review Board 1.7.22


Sqoop2: Merge HdfsSequenceExportExtractor and HdfsTextExportExtractor to one Extractor

Review Request #9692 - Created March 1, 2013 and updated

Vasanth kumar RJ
SQOOP-783
Reviewers
Sqoop
sqoop-sqoop2
Merge HdfsSequenceExportExtractor and HdfsTextExportExtractor to one HdfsExportExtractor
Passing all tests.
execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
Revision b201a8d New Change
[20] 26 lines
[+20]
27
import org.apache.sqoop.framework.configuration.ImportJobConfiguration;
27
import org.apache.sqoop.framework.configuration.ImportJobConfiguration;
28
import org.apache.sqoop.framework.configuration.OutputFormat;
28
import org.apache.sqoop.framework.configuration.OutputFormat;
29
import org.apache.sqoop.job.JobConstants;
29
import org.apache.sqoop.job.JobConstants;
30
import org.apache.sqoop.job.MapreduceExecutionError;
30
import org.apache.sqoop.job.MapreduceExecutionError;
31
import org.apache.sqoop.job.etl.Exporter;
31
import org.apache.sqoop.job.etl.Exporter;

    
   
32
import org.apache.sqoop.job.etl.HdfsExportExtractor;
32
import org.apache.sqoop.job.etl.HdfsExportPartitioner;
33
import org.apache.sqoop.job.etl.HdfsExportPartitioner;
33
import org.apache.sqoop.job.etl.HdfsSequenceImportLoader;
34
import org.apache.sqoop.job.etl.HdfsSequenceImportLoader;
34
import org.apache.sqoop.job.etl.HdfsTextExportExtractor;

   
35
import org.apache.sqoop.job.etl.HdfsTextImportLoader;
35
import org.apache.sqoop.job.etl.HdfsTextImportLoader;
36
import org.apache.sqoop.job.etl.Importer;
36
import org.apache.sqoop.job.etl.Importer;
37
import org.apache.sqoop.job.io.Data;
37
import org.apache.sqoop.job.io.Data;
38
import org.apache.sqoop.job.mr.SqoopFileOutputFormat;
38
import org.apache.sqoop.job.mr.SqoopFileOutputFormat;
39
import org.apache.sqoop.job.mr.SqoopInputFormat;
39
import org.apache.sqoop.job.mr.SqoopInputFormat;
[+20] [20] 86 lines
[+20] [+] public void prepareExportSubmission(SubmissionRequest gRequest) {
126
    MutableMapContext context = request.getFrameworkContext();
126
    MutableMapContext context = request.getFrameworkContext();
127
    context.setString(JobConstants.JOB_ETL_PARTITIONER, HdfsExportPartitioner.class.getName());
127
    context.setString(JobConstants.JOB_ETL_PARTITIONER, HdfsExportPartitioner.class.getName());
128
    context.setString(JobConstants.JOB_ETL_LOADER, exporter.getLoader().getName());
128
    context.setString(JobConstants.JOB_ETL_LOADER, exporter.getLoader().getName());
129
    context.setString(JobConstants.JOB_ETL_DESTROYER, exporter.getDestroyer().getName());
129
    context.setString(JobConstants.JOB_ETL_DESTROYER, exporter.getDestroyer().getName());
130

    
   
130

   
131
    // We should make one extractor that will be able to read all supported file types
131
    // Extractor that will be able to read all supported file types
132
    context.setString(JobConstants.JOB_ETL_EXTRACTOR, HdfsTextExportExtractor.class.getName());
132
    context.setString(JobConstants.JOB_ETL_EXTRACTOR, HdfsExportExtractor.class.getName());
133
    context.setString(JobConstants.HADOOP_INPUTDIR, jobConf.input.inputDirectory);
133
    context.setString(JobConstants.HADOOP_INPUTDIR, jobConf.input.inputDirectory);
134

    
   
134

   
135
    if(request.getExtractors() != null) {
135
    if(request.getExtractors() != null) {
136
      context.setInteger(JobConstants.JOB_ETL_EXTRACTOR_NUM, request.getExtractors());
136
      context.setInteger(JobConstants.JOB_ETL_EXTRACTOR_NUM, request.getExtractors());
137
    }
137
    }
[+20] [20] 14 lines
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportExtractor.java
New File
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java
Revision 2280828 New Change
 
execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java
Revision ae419ff New Change
 
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java
Revision fae6573 New Change
 
  1. execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java: Loading...
  2. execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportExtractor.java: Loading...
  3. execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java: Loading...
  4. execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java: Loading...
  5. execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java: Loading...