Review Board 1.7.22


<Resource leak>: Fix the file handle leaks in Symbolic & Symlink related input formats.

Review Request #7841 - Created Nov. 2, 2012 and updated

Kanna Karanam
hive
HIVE-3480
Reviewers
hive
ashutoshc, carl
hive
Noticed these file handle leaks while fixing the Symlink related unit test failures on Windows.

 
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java
Revision 1405070 New Change
[20] 60 lines
[+20] [+] public void rework(HiveConf job, MapredWork work) throws IOException {
61
          symlinks = fileSystem.listStatus(symlinkDir);
61
          symlinks = fileSystem.listStatus(symlinkDir);
62
        }
62
        }
63
        toRemovePaths.add(path);
63
        toRemovePaths.add(path);
64
        ArrayList<String> aliases = pathToAliases.remove(path);
64
        ArrayList<String> aliases = pathToAliases.remove(path);
65
        for (FileStatus symlink : symlinks) {
65
        for (FileStatus symlink : symlinks) {
66
          BufferedReader reader = new BufferedReader(new InputStreamReader(
66
          BufferedReader reader = null;

    
   
67
          try {

    
   
68
            reader = new BufferedReader(new InputStreamReader(
67
              fileSystem.open(symlink.getPath())));
69
                fileSystem.open(symlink.getPath())));
68

    
   
70

   
69
          partDesc.setInputFileFormatClass(TextInputFormat.class);
71
            partDesc.setInputFileFormatClass(TextInputFormat.class);
70

    
   
72

   
71
          String line;
73
            String line;
72
          while ((line = reader.readLine()) != null) {
74
            while ((line = reader.readLine()) != null) {
73
            // no check for the line? How to check?
75
              // no check for the line? How to check?
74
            // if the line is invalid for any reason, the job will fail.
76
              // if the line is invalid for any reason, the job will fail.
75
            toAddPathToPart.put(line, partDesc);
77
              toAddPathToPart.put(line, partDesc);
76
            pathToAliases.put(line, aliases);
78
              pathToAliases.put(line, aliases);
77
          }
79
            }

    
   
80
          } finally {

    
   
81
            org.apache.hadoop.io.IOUtils.closeStream(reader);

    
   
82
          }
78
        }
83
        }
79
      }
84
      }
80
    }
85
    }
81

    
   
86

   
82
    pathToParts.putAll(toAddPathToPart);
87
    pathToParts.putAll(toAddPathToPart);
83
    for (String toRemove : toRemovePaths) {
88
    for (String toRemove : toRemovePaths) {
84
      pathToParts.remove(toRemove);
89
      pathToParts.remove(toRemove);
85
    }
90
    }
86
  }
91
  }
87
}
92
}
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java
Revision 1405070 New Change
 
trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
Revision 1405070 New Change
 
  1. trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java: Loading...
  2. trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java: Loading...
  3. trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java: Loading...