Review Board 1.7.22


SQOOP-892 Sqoop2: Validate acceptable number of mappers and reducers

Review Request #9479 - Created Feb. 15, 2013 and submitted

Jarek Cecho
SQOOP-892
Reviewers
Sqoop
sqoop-sqoop2
I've improved the Framework validator as suggested.
I've included new unit test + tested on real cluster.
core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java
Revision 6f9a6fca7d95de7aa098c2e094fd98bb744a49b6 New Change
[20] 17 lines
[+20]
18
package org.apache.sqoop.framework;
18
package org.apache.sqoop.framework;
19

    
   
19

   
20
import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
20
import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
21
import org.apache.sqoop.framework.configuration.ExportJobConfiguration;
21
import org.apache.sqoop.framework.configuration.ExportJobConfiguration;
22
import org.apache.sqoop.framework.configuration.ImportJobConfiguration;
22
import org.apache.sqoop.framework.configuration.ImportJobConfiguration;

    
   
23
import org.apache.sqoop.framework.configuration.InputForm;

    
   
24
import org.apache.sqoop.framework.configuration.OutputForm;

    
   
25
import org.apache.sqoop.framework.configuration.ThrottlingForm;
23
import org.apache.sqoop.model.MJob;
26
import org.apache.sqoop.model.MJob;
24
import org.apache.sqoop.validation.Status;
27
import org.apache.sqoop.validation.Status;
25
import org.apache.sqoop.validation.Validation;
28
import org.apache.sqoop.validation.Validation;
26
import org.apache.sqoop.validation.Validator;
29
import org.apache.sqoop.validation.Validator;
27

    
   
30

   
[+20] [20] 24 lines
[+20] [+] public Validation validateJob(MJob.Type type, Object jobConfiguration) {
52

    
   
55

   
53
  private Validation validateExportJob(Object jobConfiguration) {
56
  private Validation validateExportJob(Object jobConfiguration) {
54
    Validation validation = new Validation(ExportJobConfiguration.class);
57
    Validation validation = new Validation(ExportJobConfiguration.class);
55
    ExportJobConfiguration configuration = (ExportJobConfiguration)jobConfiguration;
58
    ExportJobConfiguration configuration = (ExportJobConfiguration)jobConfiguration;
56

    
   
59

   
57
    if(configuration.input.inputDirectory == null || configuration.input.inputDirectory.isEmpty()) {
60
    validateInputForm(validation, configuration.input);
58
      validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");
61
    validateThrottingForm(validation, configuration.throttling);
59
    }

   
60

    
   
62

   
61
    return validation;
63
    return validation;
62
  }
64
  }
63

    
   
65

   
64
  private Validation validateImportJob(Object jobConfiguration) {
66
  private Validation validateImportJob(Object jobConfiguration) {
65
    Validation validation = new Validation(ImportJobConfiguration.class);
67
    Validation validation = new Validation(ImportJobConfiguration.class);
66
    ImportJobConfiguration configuration = (ImportJobConfiguration)jobConfiguration;
68
    ImportJobConfiguration configuration = (ImportJobConfiguration)jobConfiguration;
67

    
   
69

   
68
    if(configuration.output.outputDirectory == null || configuration.output.outputDirectory.isEmpty()) {
70
    validateOutputForm(validation, configuration.output);

    
   
71
    validateThrottingForm(validation, configuration.throttling);

    
   
72

   

    
   
73
    return validation;

    
   
74
  }

    
   
75

   

    
   
76
  private void validateInputForm(Validation validation, InputForm input) {

    
   
77
    if(input.inputDirectory == null || input.inputDirectory.isEmpty()) {

    
   
78
      validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");

    
   
79
    }

    
   
80
  }

    
   
81

   

    
   
82
  private void validateOutputForm(Validation validation, OutputForm output) {

    
   
83
    if(output.outputDirectory == null || output.outputDirectory.isEmpty()) {
69
      validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Input directory is empty");
84
      validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Input directory is empty");
70
    }
85
    }

    
   
86
  }
71

    
   
87

   
72
    return validation;
88
  private void validateThrottingForm(Validation validation, ThrottlingForm throttling) {

    
   
89
    if(throttling.extractors != null && throttling.extractors < 1) {

    
   
90
      validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");

    
   
91
    }

    
   
92

   

    
   
93
    if(throttling.loaders != null && throttling.loaders < 1) {

    
   
94
      validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");

    
   
95
    }
73
  }
96
  }
74
}
97
}
core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java
Revision d533089c4602fd7740502b6d1607ba0716acf6bb New Change
 
core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
New File
 
  1. core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java: Loading...
  2. core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java: Loading...
  3. core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java: Loading...