Review Board 1.7.22


Export dir to support subdirectories

Review Request #10646 - Created April 19, 2013 and updated

Vasanth kumar RJ
SQOOP-951
Reviewers
Sqoop
sqoop-trunk
Export dir to support subdirectories
Done
src/java/org/apache/sqoop/mapreduce/CombineFileInputFormat.java
Revision 7d2be38 New Change
[20] 208 lines
[+20] [+] protected boolean isSplitable(JobContext context, Path file) {
209
      throw new IOException("Minimum split size per node" + minSizeNode +
209
      throw new IOException("Minimum split size per node" + minSizeNode +
210
                            " cannot be smaller than minimum split " +
210
                            " cannot be smaller than minimum split " +
211
                            "size per rack " + minSizeRack);
211
                            "size per rack " + minSizeRack);
212
    }
212
    }
213

    
   
213

   

    
   
214
    List<FileStatus> fileStatusList = listStatus(job);

    
   
215
    Iterator<FileStatus> fileStatusItr = fileStatusList.iterator();

    
   
216
    //Filter only file path and remove directory paths

    
   
217
    while(fileStatusItr.hasNext()) {

    
   
218
      if(fileStatusItr.next().isDir()) {

    
   
219
        fileStatusItr.remove();

    
   
220
      }

    
   
221
    }

    
   
222

   
214
    // all the files in input set
223
    // all the files in input set
215
    Path[] paths = FileUtil.stat2Paths(
224
    Path[] paths = FileUtil.stat2Paths(
216
                     listStatus(job).toArray(new FileStatus[0]));
225
        fileStatusList.toArray(new FileStatus[0]));
217
    List<InputSplit> splits = new ArrayList<InputSplit>();
226
    List<InputSplit> splits = new ArrayList<InputSplit>();
218
    if (paths.length == 0) {
227
    if (paths.length == 0) {
219
      return splits;
228
      return splits;
220
    }
229
    }
221

    
   
230

   
[+20] [20] 492 lines
src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
Revision 1065d0b New Change
 
  1. src/java/org/apache/sqoop/mapreduce/CombineFileInputFormat.java: Loading...
  2. src/java/org/apache/sqoop/mapreduce/ExportJobBase.java: Loading...