You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2017/12/14 16:18:56 UTC
hadoop git commit: HADOOP-15085. Output streams closed with IOUtils
suppressing write errors. Contributed by Jim Brennan
Repository: hadoop
Updated Branches:
refs/heads/trunk 80db744ee -> f8af0e2fe
HADOOP-15085. Output streams closed with IOUtils suppressing write errors. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8af0e2f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8af0e2f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8af0e2f
Branch: refs/heads/trunk
Commit: f8af0e2feb9f45aeaa9711dbf93115ffb1a07e5d
Parents: 80db744
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Dec 14 10:18:08 2017 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Dec 14 10:18:08 2017 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileContext.java | 14 ++++------
.../java/org/apache/hadoop/fs/FileUtil.java | 13 +++------
.../apache/hadoop/fs/shell/CopyCommands.java | 11 ++------
.../main/java/org/apache/hadoop/io/MapFile.java | 17 ++++++------
.../org/apache/hadoop/io/nativeio/NativeIO.java | 29 ++++++++------------
.../apache/hadoop/conf/TestConfiguration.java | 7 ++---
.../apache/hadoop/io/compress/TestCodec.java | 8 ++----
.../hadoop/crypto/key/kms/server/MiniKMS.java | 8 ++----
8 files changed, 38 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index a3cc550..2a28f2b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -2121,17 +2121,13 @@ public class FileContext {
content.getPath().getName())), deleteSource, overwrite);
}
} else {
- InputStream in=null;
- OutputStream out = null;
- try {
- in = open(qSrc);
- EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of(
- CreateFlag.CREATE, CreateFlag.OVERWRITE) :
- EnumSet.of(CreateFlag.CREATE);
- out = create(qDst, createFlag);
+ EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of(
+ CreateFlag.CREATE, CreateFlag.OVERWRITE) :
+ EnumSet.of(CreateFlag.CREATE);
+ InputStream in = open(qSrc);
+ try (OutputStream out = create(qDst, createFlag)) {
IOUtils.copyBytes(in, out, conf, true);
} finally {
- IOUtils.closeStream(out);
IOUtils.closeStream(in);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 72b9615..4d971aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -1279,15 +1279,10 @@ public class FileUtil {
// Write the manifest to output JAR file
File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
- FileOutputStream fos = null;
- BufferedOutputStream bos = null;
- JarOutputStream jos = null;
- try {
- fos = new FileOutputStream(classPathJar);
- bos = new BufferedOutputStream(fos);
- jos = new JarOutputStream(bos, jarManifest);
- } finally {
- IOUtils.cleanupWithLogger(LOG, jos, bos, fos);
+ try (FileOutputStream fos = new FileOutputStream(classPathJar);
+ BufferedOutputStream bos = new BufferedOutputStream(fos)) {
+ JarOutputStream jos = new JarOutputStream(bos, jarManifest);
+ jos.close();
}
String[] jarCp = {classPathJar.getCanonicalPath(),
unexpandedWildcardClasspath.toString()};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 7b3c53e..11cb3d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.shell;
import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Iterator;
@@ -464,10 +463,8 @@ class CopyCommands {
dst.fs.create(dst.path, false).close();
}
- InputStream is = null;
- FSDataOutputStream fos = dst.fs.append(dst.path);
-
- try {
+ FileInputStream is = null;
+ try (FSDataOutputStream fos = dst.fs.append(dst.path)) {
if (readStdin) {
if (args.size() == 0) {
IOUtils.copyBytes(System.in, fos, DEFAULT_IO_LENGTH);
@@ -488,10 +485,6 @@ class CopyCommands {
if (is != null) {
IOUtils.closeStream(is);
}
-
- if (fos != null) {
- IOUtils.closeStream(fos);
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
index 2e21444..d56822f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
@@ -986,23 +986,22 @@ public class MapFile {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.getLocal(conf);
MapFile.Reader reader = null;
- MapFile.Writer writer = null;
try {
reader = new MapFile.Reader(fs, in, conf);
- writer =
- new MapFile.Writer(conf, fs, out,
- reader.getKeyClass().asSubclass(WritableComparable.class),
- reader.getValueClass());
-
WritableComparable<?> key = ReflectionUtils.newInstance(
reader.getKeyClass().asSubclass(WritableComparable.class), conf);
Writable value = ReflectionUtils.newInstance(reader.getValueClass()
.asSubclass(Writable.class), conf);
- while (reader.next(key, value)) // copy all entries
- writer.append(key, value);
+ try (MapFile.Writer writer = new MapFile.Writer(conf, fs, out,
+ reader.getKeyClass().asSubclass(WritableComparable.class),
+ reader.getValueClass())) {
+ while (reader.next(key, value)) { // copy all entries
+ writer.append(key, value);
+ }
+ }
} finally {
- IOUtils.cleanupWithLogger(LOG, writer, reader);
+ IOUtils.cleanupWithLogger(LOG, reader);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index f601edd..adc74bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -956,28 +956,23 @@ public class NativeIO {
if (nativeLoaded && Shell.WINDOWS) {
copyFileUnbuffered0(src.getAbsolutePath(), dst.getAbsolutePath());
} else {
- FileInputStream fis = null;
- FileOutputStream fos = null;
+ FileInputStream fis = new FileInputStream(src);
FileChannel input = null;
- FileChannel output = null;
try {
- fis = new FileInputStream(src);
- fos = new FileOutputStream(dst);
input = fis.getChannel();
- output = fos.getChannel();
- long remaining = input.size();
- long position = 0;
- long transferred = 0;
- while (remaining > 0) {
- transferred = input.transferTo(position, remaining, output);
- remaining -= transferred;
- position += transferred;
+ try (FileOutputStream fos = new FileOutputStream(dst);
+ FileChannel output = fos.getChannel()) {
+ long remaining = input.size();
+ long position = 0;
+ long transferred = 0;
+ while (remaining > 0) {
+ transferred = input.transferTo(position, remaining, output);
+ remaining -= transferred;
+ position += transferred;
+ }
}
} finally {
- IOUtils.cleanupWithLogger(LOG, output);
- IOUtils.cleanupWithLogger(LOG, fos);
- IOUtils.cleanupWithLogger(LOG, input);
- IOUtils.cleanupWithLogger(LOG, fis);
+ IOUtils.cleanupWithLogger(LOG, input, fis);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 52215da..78dcdd6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -56,7 +56,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -385,11 +384,9 @@ public class TestConfiguration {
Configuration conf = new Configuration(false);
conf.addResource(new Path(CONFIG_MULTI_BYTE));
assertEquals(value, conf.get(name));
- FileOutputStream fos = new FileOutputStream(CONFIG_MULTI_BYTE_SAVED);
- try {
+ try (FileOutputStream fos =
+ new FileOutputStream(CONFIG_MULTI_BYTE_SAVED)) {
conf.writeXml(fos);
- } finally {
- IOUtils.closeStream(fos);
}
conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
index 133ff9a..94ff7a8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
@@ -362,11 +362,10 @@ public class TestCodec {
final Path file = new Path(wd, "test" + codec.getDefaultExtension());
final byte[] b = new byte[REC_SIZE];
final Base64 b64 = new Base64(0, null);
- DataOutputStream fout = null;
Compressor cmp = CodecPool.getCompressor(codec);
- try {
- fout = new DataOutputStream(codec.createOutputStream(
- fs.create(file, true), cmp));
+ try (DataOutputStream fout =
+ new DataOutputStream(codec.createOutputStream(fs.create(file,
+ true), cmp))) {
final DataOutputBuffer dob = new DataOutputBuffer(REC_SIZE * 4 / 3 + 4);
int seq = 0;
while (infLen > 0) {
@@ -382,7 +381,6 @@ public class TestCodec {
}
LOG.info("Wrote " + seq + " records to " + file);
} finally {
- IOUtils.cleanupWithLogger(LOG, fout);
CodecPool.returnCompressor(cmp);
}
return file;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8af0e2f/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 0a7ff41..faa8fa2 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -105,15 +105,11 @@ public class MiniKMS {
private void copyResource(String inputResourceName, File outputFile) throws
IOException {
- InputStream is = null;
- OutputStream os = null;
- try {
- is = ThreadUtil.getResourceAsStream(inputResourceName);
- os = new FileOutputStream(outputFile);
+ InputStream is = ThreadUtil.getResourceAsStream(inputResourceName);
+ try (OutputStream os = new FileOutputStream(outputFile)) {
IOUtils.copy(is, os);
} finally {
IOUtils.closeQuietly(is);
- IOUtils.closeQuietly(os);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org