You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/10/24 05:52:41 UTC
[21/50] [abbrv] hadoop git commit: HDFS-13983. TestOfflineImageViewer
crashes in windows. Contributed by Vinayakumar B.
HDFS-13983. TestOfflineImageViewer crashes in windows. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f069d38c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f069d38c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f069d38c
Branch: refs/heads/HDFS-12943
Commit: f069d38c8d3c0bfa91b70a60e4e556ec708fc411
Parents: e2a9fa8
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Oct 19 17:16:04 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Oct 19 17:16:04 2018 -0700
----------------------------------------------------------------------
.../OfflineImageViewerPB.java | 31 +++++---
.../TestOfflineImageViewer.java | 82 +++++++++++---------
2 files changed, 66 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f069d38c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index 34a85a6..782ace8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
/**
@@ -135,7 +136,7 @@ public class OfflineImageViewerPB {
*/
public static void main(String[] args) throws Exception {
int status = run(args);
- System.exit(status);
+ ExitUtil.terminate(status);
}
public static int run(String[] args) throws Exception {
@@ -175,19 +176,24 @@ public class OfflineImageViewerPB {
String tempPath = cmd.getOptionValue("t", "");
Configuration conf = new Configuration();
- try (PrintStream out = outputFile.equals("-") ?
- System.out : new PrintStream(outputFile, "UTF-8")) {
+ PrintStream out = null;
+ try {
+ out = outputFile.equals("-") || "REVERSEXML".equalsIgnoreCase(processor) ?
+ System.out : new PrintStream(outputFile, "UTF-8");
switch (StringUtils.toUpperCase(processor)) {
case "FILEDISTRIBUTION":
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
boolean formatOutput = cmd.hasOption("format");
- new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
- .visit(new RandomAccessFile(inputFile, "r"));
+ try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
+ new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
+ .visit(r);
+ }
break;
case "XML":
- new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
- "r"));
+ try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
+ new PBImageXmlWriter(conf, out).visit(r);
+ }
break;
case "REVERSEXML":
try {
@@ -196,7 +202,7 @@ public class OfflineImageViewerPB {
System.err.println("OfflineImageReconstructor failed: "
+ e.getMessage());
e.printStackTrace(System.err);
- System.exit(1);
+ ExitUtil.terminate(1);
}
break;
case "WEB":
@@ -208,8 +214,9 @@ public class OfflineImageViewerPB {
break;
case "DELIMITED":
try (PBImageDelimitedTextWriter writer =
- new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
- writer.visit(new RandomAccessFile(inputFile, "r"));
+ new PBImageDelimitedTextWriter(out, delimiter, tempPath);
+ RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
+ writer.visit(r);
}
break;
default:
@@ -223,6 +230,10 @@ public class OfflineImageViewerPB {
} catch (IOException e) {
System.err.println("Encountered exception. Exiting: " + e.getMessage());
e.printStackTrace(System.err);
+ } finally {
+ if (out != null && out != System.out) {
+ out.close();
+ }
}
return -1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f069d38c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 16b35dc..5574d7b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -355,8 +355,10 @@ public class TestOfflineImageViewer {
File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
- new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
- .visit(new RandomAccessFile(truncatedFile, "r"));
+ try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) {
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
+ .visit(r);
+ }
}
private void copyPartOfFile(File src, File dest) throws IOException {
@@ -377,38 +379,41 @@ public class TestOfflineImageViewer {
@Test
public void testFileDistributionCalculator() throws IOException {
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- PrintStream o = new PrintStream(output);
- new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
- .visit(new RandomAccessFile(originalFsimage, "r"));
- o.close();
-
- String outputString = output.toString();
- Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
- Matcher matcher = p.matcher(outputString);
- assertTrue(matcher.find() && matcher.groupCount() == 1);
- int totalFiles = Integer.parseInt(matcher.group(1));
- assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
-
- p = Pattern.compile("totalDirectories = (\\d+)\n");
- matcher = p.matcher(outputString);
- assertTrue(matcher.find() && matcher.groupCount() == 1);
- int totalDirs = Integer.parseInt(matcher.group(1));
- // totalDirs includes root directory
- assertEquals(dirCount + 1, totalDirs);
-
- FileStatus maxFile = Collections.max(writtenFiles.values(),
- new Comparator<FileStatus>() {
- @Override
- public int compare(FileStatus first, FileStatus second) {
- return first.getLen() < second.getLen() ? -1 :
- ((first.getLen() == second.getLen()) ? 0 : 1);
- }
- });
- p = Pattern.compile("maxFileSize = (\\d+)\n");
- matcher = p.matcher(output.toString("UTF-8"));
- assertTrue(matcher.find() && matcher.groupCount() == 1);
- assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
+ try (ByteArrayOutputStream output = new ByteArrayOutputStream();
+ PrintStream o = new PrintStream(output);
+ RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
+ .visit(r);
+ o.close();
+
+ String outputString = output.toString();
+ Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
+ Matcher matcher = p.matcher(outputString);
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ int totalFiles = Integer.parseInt(matcher.group(1));
+ assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
+
+ p = Pattern.compile("totalDirectories = (\\d+)\n");
+ matcher = p.matcher(outputString);
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ int totalDirs = Integer.parseInt(matcher.group(1));
+ // totalDirs includes root directory
+ assertEquals(dirCount + 1, totalDirs);
+
+ FileStatus maxFile = Collections.max(writtenFiles.values(),
+ new Comparator<FileStatus>() {
+ @Override
+ public int compare(FileStatus first, FileStatus second) {
+ return first.getLen() < second.getLen() ?
+ -1 :
+ ((first.getLen() == second.getLen()) ? 0 : 1);
+ }
+ });
+ p = Pattern.compile("maxFileSize = (\\d+)\n");
+ matcher = p.matcher(output.toString("UTF-8"));
+ assertTrue(matcher.find() && matcher.groupCount() == 1);
+ assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
+ }
}
@Test
@@ -514,7 +519,9 @@ public class TestOfflineImageViewer {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
- v.visit(new RandomAccessFile(originalFsimage, "r"));
+ try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
+ v.visit(r);
+ }
SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser parser = spf.newSAXParser();
final String xml = output.toString();
@@ -667,10 +674,11 @@ public class TestOfflineImageViewer {
final String DELIMITER = "\t";
ByteArrayOutputStream output = new ByteArrayOutputStream();
- try (PrintStream o = new PrintStream(output)) {
+ try (PrintStream o = new PrintStream(output);
+ RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
PBImageDelimitedTextWriter v =
new PBImageDelimitedTextWriter(o, DELIMITER, db);
- v.visit(new RandomAccessFile(originalFsimage, "r"));
+ v.visit(r);
}
Set<String> fileNames = new HashSet<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org