You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2012/06/01 04:31:30 UTC
svn commit: r1344970 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/
src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/
Author: eli
Date: Fri Jun 1 02:31:29 2012
New Revision: 1344970
URL: http://svn.apache.org/viewvc?rev=1344970&view=rev
Log:
HDFS-3486. offlineimageviewer can't read fsimage files that contain persistent delegation tokens. Contributed by Colin Patrick McCabe
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1344970&r1=1344969&r2=1344970&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jun 1 02:31:29 2012
@@ -273,6 +273,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3484. hdfs fsck doesn't work if NN HTTP address is set to
0.0.0.0 even if NN RPC address is configured. (atm via eli)
+ HDFS-3486. offlineimageviewer can't read fsimage files that contain
+ persistent delegation tokens. (Colin Patrick McCabe via eli)
+
Release 2.0.0-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1344970&r1=1344969&r2=1344970&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Fri Jun 1 02:31:29 2012
@@ -31,11 +31,13 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
+import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.xml.sax.helpers.AttributesImpl;
/**
* ImageLoaderCurrent processes Hadoop FSImage files and walks over
@@ -220,9 +222,29 @@ class ImageLoaderCurrent implements Imag
for(int i=0; i<numDTokens; i++){
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
- v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER, id.toString());
+ long expiryTime = in.readLong();
+ v.visitEnclosingElement(ImageElement.DELEGATION_TOKEN_IDENTIFIER);
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_KIND,
+ id.getKind().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_SEQNO,
+ id.getSequenceNumber());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_OWNER,
+ id.getOwner().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER,
+ id.getRenewer().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_REALUSER,
+ id.getRealUser().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
+ id.getIssueDate());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
+ id.getMaxDate());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
+ expiryTime);
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
+ id.getMasterKeyId());
+ v.leaveEnclosingElement(); // DELEGATION_TOKEN_IDENTIFIER
}
- v.leaveEnclosingElement();
+ v.leaveEnclosingElement(); // DELEGATION_TOKENS
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java?rev=1344970&r1=1344969&r2=1344970&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java Fri Jun 1 02:31:29 2012
@@ -71,7 +71,15 @@ abstract class ImageVisitor {
NUM_DELEGATION_TOKENS,
DELEGATION_TOKENS,
DELEGATION_TOKEN_IDENTIFIER,
- DELEGATION_TOKEN_EXPIRY_TIME,
+ DELEGATION_TOKEN_IDENTIFIER_KIND,
+ DELEGATION_TOKEN_IDENTIFIER_SEQNO,
+ DELEGATION_TOKEN_IDENTIFIER_OWNER,
+ DELEGATION_TOKEN_IDENTIFIER_RENEWER,
+ DELEGATION_TOKEN_IDENTIFIER_REALUSER,
+ DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
+ DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
+ DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
+ DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
TRANSACTION_ID
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java?rev=1344970&r1=1344969&r2=1344970&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java Fri Jun 1 02:31:29 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.IOException;
+import java.util.Date;
/**
* IndentedImageVisitor walks over an FSImage and displays its structure
@@ -58,6 +59,16 @@ class IndentedImageVisitor extends TextW
write(element + " = " + value + "\n");
}
+ void visit(ImageElement element, long value) throws IOException {
+ if ((element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME) ||
+ (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE) ||
+ (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE)) {
+ visit(element, new Date(value).toString());
+ } else {
+ visit(element, Long.toString(value));
+ }
+ }
+
@Override
void visitEnclosingElement(ImageElement element) throws IOException {
printIndents();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1344970&r1=1344969&r2=1344970&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Fri Jun 1 02:31:29 2012
@@ -18,6 +18,10 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.BufferedReader;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.token.Token;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
@@ -29,15 +33,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Set;
-import junit.framework.TestCase;
+import org.junit.*;
+import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
@@ -52,12 +60,15 @@ import org.apache.hadoop.hdfs.HdfsConfig
* * confirm it correctly bails on malformed image files, in particular, a
* file that ends suddenly.
*/
-public class TestOfflineImageViewer extends TestCase {
+public class TestOfflineImageViewer {
+ private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
private static final int NUM_DIRS = 3;
private static final int FILES_PER_DIR = 4;
+ private static final String TEST_RENEWER = "JobTracker";
+ private static File originalFsimage = null;
// Elements of lines of ls-file output to be compared to FileStatus instance
- private class LsElements {
+ private static class LsElements {
public String perms;
public int replication;
public String username;
@@ -67,43 +78,28 @@ public class TestOfflineImageViewer exte
}
// namespace as written to dfs, to be compared with viewer's output
- final HashMap<String, FileStatus> writtenFiles
- = new HashMap<String, FileStatus>();
-
+ final static HashMap<String, FileStatus> writtenFiles =
+ new HashMap<String, FileStatus>();
private static String ROOT = System.getProperty("test.build.data",
"build/test/data");
- // Main entry point into testing. Necessary since we only want to generate
- // the fsimage file once and use it for multiple tests.
- public void testOIV() throws Exception {
- File originalFsimage = null;
- try {
- originalFsimage = initFsimage();
- assertNotNull("originalFsImage shouldn't be null", originalFsimage);
-
- // Tests:
- outputOfLSVisitor(originalFsimage);
- outputOfFileDistributionVisitor(originalFsimage);
-
- unsupportedFSLayoutVersion(originalFsimage);
-
- truncatedFSImage(originalFsimage);
-
- } finally {
- if(originalFsimage != null && originalFsimage.exists())
- originalFsimage.delete();
- }
- }
-
// Create a populated namespace for later testing. Save its contents to a
// data structure and store its fsimage location.
- private File initFsimage() throws IOException {
+ // We only want to generate the fsimage file once and use it for
+ // multiple tests.
+ @BeforeClass
+ public static void createOriginalFSImage() throws IOException {
MiniDFSCluster cluster = null;
- File orig = null;
try {
Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ conf.set("hadoop.security.auth_to_local",
+ "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ cluster.waitActive();
FileSystem hdfs = cluster.getFileSystem();
int filesize = 256;
@@ -123,34 +119,49 @@ public class TestOfflineImageViewer exte
}
}
+ // Get delegation tokens so we log the delegation token op
+ List<Token<?>> delegationTokens =
+ hdfs.getDelegationTokens(TEST_RENEWER);
+ for (Token<?> t : delegationTokens) {
+ LOG.debug("got token " + t);
+ }
+
// Write results to the fsimage file
cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
cluster.getNameNodeRpc().saveNamespace();
// Determine location of fsimage file
- orig = FSImageTestUtil.findLatestImageFile(
+ originalFsimage = FSImageTestUtil.findLatestImageFile(
FSImageTestUtil.getFSImage(
cluster.getNameNode()).getStorage().getStorageDir(0));
- if (orig == null) {
- fail("Didn't generate or can't find fsimage");
+ if (originalFsimage == null) {
+ throw new RuntimeException("Didn't generate or can't find fsimage");
}
+ LOG.debug("original FS image file is " + originalFsimage);
} finally {
if(cluster != null)
cluster.shutdown();
}
- return orig;
+ }
+
+ @AfterClass
+ public static void deleteOriginalFSImage() throws IOException {
+ if(originalFsimage != null && originalFsimage.exists()) {
+ originalFsimage.delete();
+ }
}
// Convenience method to generate a file status from file system for
// later comparison
- private FileStatus pathToFileEntry(FileSystem hdfs, String file)
+ private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
throws IOException {
return hdfs.getFileStatus(new Path(file));
}
-
+
// Verify that we can correctly generate an ls-style output for a valid
// fsimage
- private void outputOfLSVisitor(File originalFsimage) throws IOException {
+ @Test
+ public void outputOfLSVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
File outputFile = new File(ROOT, "/basicCheckOutput");
@@ -169,12 +180,13 @@ public class TestOfflineImageViewer exte
if(testFile.exists()) testFile.delete();
if(outputFile.exists()) outputFile.delete();
}
- System.out.println("Correctly generated ls-style output.");
+ LOG.debug("Correctly generated ls-style output.");
}
// Confirm that attempting to read an fsimage file with an unsupported
// layout results in an error
- public void unsupportedFSLayoutVersion(File originalFsimage) throws IOException {
+ @Test
+ public void unsupportedFSLayoutVersion() throws IOException {
File testFile = new File(ROOT, "/invalidLayoutVersion");
File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
@@ -190,7 +202,7 @@ public class TestOfflineImageViewer exte
} catch(IOException e) {
if(!e.getMessage().contains(Integer.toString(badVersionNum)))
throw e; // wasn't error we were expecting
- System.out.println("Correctly failed at reading bad image version.");
+ LOG.debug("Correctly failed at reading bad image version.");
}
} finally {
if(testFile.exists()) testFile.delete();
@@ -199,7 +211,8 @@ public class TestOfflineImageViewer exte
}
// Verify that image viewer will bail on a file that ends unexpectedly
- private void truncatedFSImage(File originalFsimage) throws IOException {
+ @Test
+ public void truncatedFSImage() throws IOException {
File testFile = new File(ROOT, "/truncatedFSImage");
File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
try {
@@ -213,7 +226,7 @@ public class TestOfflineImageViewer exte
oiv.go();
fail("Managed to process a truncated fsimage file");
} catch (EOFException e) {
- System.out.println("Correctly handled EOF");
+ LOG.debug("Correctly handled EOF");
}
} finally {
@@ -365,7 +378,8 @@ public class TestOfflineImageViewer exte
}
}
- private void outputOfFileDistributionVisitor(File originalFsimage) throws IOException {
+ @Test
+ public void outputOfFileDistributionVisitor() throws IOException {
File testFile = new File(ROOT, "/basicCheck");
File outputFile = new File(ROOT, "/fileDistributionCheckOutput");
@@ -392,4 +406,66 @@ public class TestOfflineImageViewer exte
}
assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR);
}
+
+ private static class TestImageVisitor extends ImageVisitor {
+ private List<String> delegationTokenRenewers = new LinkedList<String>();
+ TestImageVisitor() {
+ }
+
+ List<String> getDelegationTokenRenewers() {
+ return delegationTokenRenewers;
+ }
+
+ @Override
+ void start() throws IOException {
+ }
+
+ @Override
+ void finish() throws IOException {
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) {
+ delegationTokenRenewers.add(value);
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element, ImageElement key,
+ String value) throws IOException {
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ }
+ }
+
+ @Test
+ public void outputOfTestVisitor() throws IOException {
+ File testFile = new File(ROOT, "/basicCheck");
+
+ try {
+ copyFile(originalFsimage, testFile);
+ TestImageVisitor v = new TestImageVisitor();
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true);
+ oiv.go();
+
+ // Validated stored delegation token identifiers.
+ List<String> dtrs = v.getDelegationTokenRenewers();
+ assertEquals(1, dtrs.size());
+ assertEquals(TEST_RENEWER, dtrs.get(0));
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ }
+ LOG.debug("Passed TestVisitor validation.");
+ }
}