You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/11/08 17:31:30 UTC
svn commit: r1407158 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java
java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java
test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
Author: hashutosh
Date: Thu Nov 8 16:31:30 2012
New Revision: 1407158
URL: http://svn.apache.org/viewvc?rev=1407158&view=rev
Log:
HIVE-3480 : <Resource leak>: Fix the file handle leaks in Symbolic & Symlink related input formats. (Kanna Karanam via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java?rev=1407158&r1=1407157&r2=1407158&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java Thu Nov 8 16:31:30 2012
@@ -63,17 +63,22 @@ public class SymbolicInputFormat impleme
toRemovePaths.add(path);
ArrayList<String> aliases = pathToAliases.remove(path);
for (FileStatus symlink : symlinks) {
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- fileSystem.open(symlink.getPath())));
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new InputStreamReader(
+ fileSystem.open(symlink.getPath())));
- partDesc.setInputFileFormatClass(TextInputFormat.class);
+ partDesc.setInputFileFormatClass(TextInputFormat.class);
- String line;
- while ((line = reader.readLine()) != null) {
- // no check for the line? How to check?
- // if the line is invalid for any reason, the job will fail.
- toAddPathToPart.put(line, partDesc);
- pathToAliases.put(line, aliases);
+ String line;
+ while ((line = reader.readLine()) != null) {
+ // no check for the line? How to check?
+ // if the line is invalid for any reason, the job will fail.
+ toAddPathToPart.put(line, partDesc);
+ pathToAliases.put(line, aliases);
+ }
+ } finally {
+ org.apache.hadoop.io.IOUtils.closeStream(reader);
}
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java?rev=1407158&r1=1407157&r2=1407158&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java Thu Nov 8 16:31:30 2012
@@ -196,15 +196,18 @@ public class SymlinkTextInputFormat exte
// Read paths from each symlink file.
for (FileStatus symlink : symlinks) {
- BufferedReader reader =
- new BufferedReader(
- new InputStreamReader(
- fileSystem.open(symlink.getPath())));
-
- String line;
- while ((line = reader.readLine()) != null) {
- targetPaths.add(new Path(line));
- symlinkPaths.add(symlink.getPath());
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(
+ new InputStreamReader(
+ fileSystem.open(symlink.getPath())));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ targetPaths.add(new Path(line));
+ symlinkPaths.add(symlink.getPath());
+ }
+ } finally {
+ org.apache.hadoop.io.IOUtils.closeStream(reader);
}
}
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java?rev=1407158&r1=1407157&r2=1407158&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java Thu Nov 8 16:31:30 2012
@@ -154,6 +154,7 @@ public class TestSymlinkTextInputFormat
+ " failed with exit code= " + ecode);
}
+ tblCreated = true;
String loadFileCommand = "LOAD DATA LOCAL INPATH '" +
new Path(symlinkDir, "symlink_file").toString() + "' INTO TABLE " + tblName;
@@ -263,6 +264,7 @@ public class TestSymlinkTextInputFormat
while (reader.next(key, value)) {
received.add(value.toString());
}
+ reader.close();
}
List<String> expected = new ArrayList<String>();
@@ -307,6 +309,7 @@ public class TestSymlinkTextInputFormat
while (reader.next(key, value)) {
received.add(value.toString());
}
+ reader.close();
}
List<String> expected = new ArrayList<String>();