Review Board 1.7.22


SQOOP-986 Sqoop2: Add detection of local mode to mapreduce submission engine

Review Request #10452 - Created April 14, 2013 and submitted

Jarek Cecho
SQOOP-986
Reviewers
Sqoop
sqoop-sqoop2
I've added local job runner detection to the mapreduce submission engine.
Integration tests seems to be passing in both hadoop profiles (100, 200) and I've also tried running this patch on real hadoop clusters (both 1.x and 2.x).
submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
Revision 77f30ea3b4172217c3e7c40aa4adfee87c754427 New Change
[20] 109 lines
[+20] [+] public boolean accept(File dir, String name) {
110
    try {
110
    try {
111
      jobClient = new JobClient(new JobConf(globalConfiguration));
111
      jobClient = new JobClient(new JobConf(globalConfiguration));
112
    } catch (IOException e) {
112
    } catch (IOException e) {
113
      throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0002, e);
113
      throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0002, e);
114
    }
114
    }

    
   
115

   

    
   
116
    if(isLocal()) {

    
   
117
      LOG.info("Detected MapReduce local mode, some methods might not work correctly.");

    
   
118
    }
115
  }
119
  }
116

    
   
120

   
117
  /**
121
  /**
118
   * {@inheritDoc}
122
   * {@inheritDoc}
119
   */
123
   */
[+20] [20] 124 lines
[+20] [+] public boolean submit(SubmissionRequest generalRequest) {
244

    
   
248

   
245
      job.setOutputFormatClass(request.getOutputFormatClass());
249
      job.setOutputFormatClass(request.getOutputFormatClass());
246
      job.setOutputKeyClass(request.getOutputKeyClass());
250
      job.setOutputKeyClass(request.getOutputKeyClass());
247
      job.setOutputValueClass(request.getOutputValueClass());
251
      job.setOutputValueClass(request.getOutputValueClass());
248

    
   
252

   

    
   
253
      // If we're in local mode than wait on completion. Local job runner do not

    
   
254
      // seems to be exposing API to get previously submitted job which makes

    
   
255
      // other methods of the submission engine quite useless.

    
   
256
      if(isLocal()) {

    
   
257
        job.waitForCompletion(true);

    
   
258
      } else {
249
      job.submit();
259
        job.submit();

    
   
260
      }
250

    
   
261

   
251
      String jobId = job.getJobID().toString();
262
      String jobId = job.getJobID().toString();
252
      request.getSummary().setExternalId(jobId);
263
      request.getSummary().setExternalId(jobId);
253
      request.getSummary().setExternalLink(job.getTrackingURL());
264
      request.getSummary().setExternalLink(job.getTrackingURL());
254

    
   
265

   
[+20] [20] 140 lines
[+20] [+] private Counters convertMapreduceCounters(org.apache.hadoop.mapred.Counters hadoopCounters) {
395
    }
406
    }
396

    
   
407

   
397
    return sqoopCounters;
408
    return sqoopCounters;
398
  }
409
  }
399

    
   
410

   

    
   
411
  /**

    
   
412
   * Detect MapReduce local mode.

    
   
413
   *

    
   
414
   * @return True if we're running in local mode

    
   
415
   */

    
   
416
  private boolean isLocal() {

    
   
417
    return "local".equals(globalConfiguration.get("mapreduce.jobtracker.address"))

    
   
418
        || "local".equals(globalConfiguration.get("mapred.job.tracker"));

    
   
419
  }
400
}
420
}
  1. submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java: Loading...