You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/03/20 23:18:27 UTC
[16/40] hadoop git commit: HDFS-11420. Edit file should not be
processed by the same type processor in OfflineEditsViewer. Contributed by
Yiqun Lin.
HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4416a07c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4416a07c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4416a07c
Branch: refs/heads/HADOOP-13345
Commit: 4416a07c9c57af01ffbd2e43afcde2a1ba94c736
Parents: 2d0e24e
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Mar 15 18:05:03 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Mar 15 18:05:03 2017 +0800
----------------------------------------------------------------------
.../offlineEditsViewer/OfflineEditsViewer.java | 19 +++++++++++++++++--
.../src/site/markdown/HdfsEditsViewer.md | 2 ++
.../TestOfflineEditsViewer.java | 18 ++++++++++++++++++
3 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4416a07c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
index 107881f..f075ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -55,7 +56,9 @@ public class OfflineEditsViewer extends Configured implements Tool {
"Required command line arguments:\n" +
"-i,--inputFile <arg> edits file to process, xml (case\n" +
" insensitive) extension means XML format,\n" +
- " any other filename means binary format\n" +
+ " any other filename means binary format.\n" +
+ " XML/Binary format input file is not allowed\n" +
+ " to be processed by the same type processor.\n" +
"-o,--outputFile <arg> Name of output file. If the specified\n" +
" file exists, it will be overwritten,\n" +
" format of the file is determined\n" +
@@ -132,12 +135,24 @@ public class OfflineEditsViewer extends Configured implements Tool {
System.out.println("input [" + inputFileName + "]");
System.out.println("output [" + outputFileName + "]");
}
+
+ boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml");
+ if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) {
+ System.err.println("XML format input file is not allowed"
+ + " to be processed by XML processor.");
+ return -1;
+ } else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) {
+ System.err.println("Binary format input file is not allowed"
+ + " to be processed by Binary processor.");
+ return -1;
+ }
+
try {
if (visitor == null) {
visitor = OfflineEditsVisitorFactory.getEditsVisitor(
outputFileName, processor, flags.getPrintToScreen());
}
- boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml");
+
OfflineEditsLoader loader = OfflineEditsLoaderFactory.
createLoader(visitor, inputFileName, xmlInput, flags);
loader.loadEdits();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4416a07c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
index 5e069bb..4ab07ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
@@ -30,6 +30,8 @@ Input formats supported:
2. **xml**: XML format, as produced by xml processor, used if filename
has `.xml` (case insensitive) extension
+Note: XML/Binary format input file is not allowed to be processed by the same type processor.
+
The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file):
1. **binary**: native binary format that Hadoop uses internally
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4416a07c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
index a21bc8f..bbad73c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
@@ -338,4 +338,22 @@ public class TestOfflineEditsViewer {
}
}
}
+
+ @Test
+ public void testProcessorWithSameTypeFormatFile() throws IOException {
+ String edits = nnHelper.generateEdits();
+ LOG.info("Generated edits=" + edits);
+ String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath();
+ String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath();
+ String editsReparsedXml = folder.newFile("editsReparsed.xml")
+ .getAbsolutePath();
+
+ // Binary format input file is not allowed to be processed
+ // by Binary processor.
+ assertEquals(-1, runOev(edits, binaryEdits, "binary", false));
+ // parse to XML then back to XML
+ assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
+ // XML format input file is not allowed to be processed by XML processor.
+ assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org