You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2009/04/22 01:39:34 UTC
svn commit: r767331 [2/2] - in /hadoop/core/trunk: ./ bin/
src/docs/src/documentation/content/xdocs/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/
src/test/org/apache/hadoop/hdfs/tools/ src/t...
Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=767331&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Tue Apr 21 23:39:33 2009
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
+
+/**
+ * Test function of OfflineImageViewer by:
+ * * confirming it can correctly process a valid fsimage file and that
+ * the processing generates a correct representation of the namespace
+ * * confirming it correctly fails to process an fsimage file with a layout
+ * version it shouldn't be able to handle
+ * * confirm it correctly bails on malformed image files, in particular, a
+ * file that ends suddenly.
+ */
+public class TestOfflineImageViewer extends TestCase {
+
+ // Elements of lines of ls-file output to be compared to FileStatus instance
+ private class LsElements {
+ public String perms;
+ public int replication;
+ public String username;
+ public String groupname;
+ public long filesize;
+ public char dir; // d if dir, - otherwise
+ }
+
+ // namespace as written to dfs, to be compared with viewer's output
+ final HashMap<String, FileStatus> writtenFiles
+ = new HashMap<String, FileStatus>();
+
+
+ private static String ROOT = System.getProperty("test.build.data","/tmp");
+
+ // Main entry point into testing. Necessary since we only want to generate
+ // the fsimage file once and use it for multiple tests.
+ public void testOIV() {
+ File originalFsimage = null;
+ try {
+ originalFsimage = initFsimage();
+ assertNotNull("originalFsImage shouldn't be null", originalFsimage);
+
+ // Tests:
+ outputOfLSVisitor(originalFsimage);
+
+ unsupportedFSLayoutVersion(originalFsimage);
+
+ truncatedFSImage(originalFsimage);
+
+ } finally {
+ if(originalFsimage != null && originalFsimage.exists())
+ originalFsimage.delete();
+ }
+ }
+
+ // Create a populated namespace for later testing. Save its contents to a
+ // data structure and store its fsimage location.
+ private File initFsimage() {
+ MiniDFSCluster cluster = null;
+ File orig = null;
+ try {
+ Configuration conf = new Configuration();
+ cluster = new MiniDFSCluster(conf, 4, true, null);
+ FileSystem hdfs = cluster.getFileSystem();
+
+ int numDirs = 3;
+ int numFilesPerDir = 4;
+ int filesize = 256;
+
+ // Create a reasonable namespace
+ for(int i = 0; i < numDirs; i++) {
+ Path dir = new Path("/dir" + i);
+ hdfs.mkdirs(dir);
+ writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
+ for(int j = 0; j < numFilesPerDir; j++) {
+ Path file = new Path(dir, "file" + j);
+ FSDataOutputStream o = hdfs.create(file);
+ o.write(new byte[ filesize++ ]);
+ o.close();
+
+ writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString()));
+ }
+ }
+
+ // Write results to the fsimage file
+ cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNode().saveNamespace();
+
+ // Determine location of fsimage file
+ File [] files = cluster.getNameDirs().toArray(new File[0]);
+ orig = new File(files[0], "current/fsimage");
+
+ if(!orig.exists())
+ fail("Didn't generate or can't find fsimage.");
+
+ } catch (IOException e) {
+ fail("Failed trying to generate fsimage file: " + e.getMessage());
+ } finally {
+ if(cluster != null)
+ cluster.shutdown();
+ }
+ return orig;
+ }
+
+ // Convenience method to generate a file status from file system for
+ // later comparison
+ private FileStatus pathToFileEntry(FileSystem hdfs, String file)
+ throws IOException {
+ return hdfs.getFileStatus(new Path(file));
+ }
+
+ // Verify that we can correctly generate an ls-style output for a valid
+ // fsimage
+ private void outputOfLSVisitor(File originalFsimage) {
+ File testFile = new File(ROOT, "/basicCheck");
+ File outputFile = new File(ROOT, "/basicCheckOutput");
+
+ try {
+ copyFile(originalFsimage, testFile);
+
+ ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+ oiv.go();
+
+ HashMap<String, LsElements> fileOutput = readLsfile(outputFile);
+
+ compareNamespaces(writtenFiles, fileOutput);
+ } catch (IOException e) {
+ fail("Failed reading valid file: " + e.getMessage());
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ System.out.println("Correctly generated ls-style output.");
+ }
+
+ // Confirm that attempting to read an fsimage file with an unsupported
+ // layout results in an error
+ public void unsupportedFSLayoutVersion(File originalFsimage) {
+ File testFile = new File(ROOT, "/invalidLayoutVersion");
+ File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
+
+ try {
+ int badVersionNum = -432;
+ changeLayoutVersion(originalFsimage, testFile, badVersionNum);
+ ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+ try {
+ oiv.go();
+ fail("Shouldn't be able to read invalid laytout version");
+ } catch(IOException e) {
+ if(!e.getMessage().contains(Integer.toString(badVersionNum)))
+ throw e; // wasn't error we were expecting
+ System.out.println("Correctly failed at reading bad image version.");
+ }
+ } catch (IOException e) {
+ fail("Problem testing unsupported layout version: " + e.getMessage());
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ }
+
+ // Verify that image viewer will bail on a file that ends unexpectedly
+ private void truncatedFSImage(File originalFsimage) {
+ File testFile = new File(ROOT, "/truncatedFSImage");
+ File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
+ try {
+ copyPartOfFile(originalFsimage, testFile);
+ assertTrue("Created truncated fsimage", testFile.exists());
+
+ ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+ try {
+ oiv.go();
+ fail("Managed to process a truncated fsimage file");
+ } catch (EOFException e) {
+ System.out.println("Correctly handled EOF");
+ }
+
+ } catch (IOException e) {
+ fail("Failed testing truncatedFSImage: " + e.getMessage());
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ }
+
+ // Test that our ls file has all the same compenents of the original namespace
+ private void compareNamespaces(HashMap<String, FileStatus> written,
+ HashMap<String, LsElements> fileOutput) {
+ assertEquals( "Should be the same number of files in both, plus one for root"
+ + " in fileoutput", fileOutput.keySet().size(),
+ written.keySet().size() + 1);
+ Set<String> inFile = fileOutput.keySet();
+
+ // For each line in the output file, verify that the namespace had a
+ // filestatus counterpart
+ for (String path : inFile) {
+ if (path.equals("/")) // root's not included in output from system call
+ continue;
+
+ assertTrue("Path in file (" + path + ") was written to fs", written
+ .containsKey(path));
+
+ compareFiles(written.get(path), fileOutput.get(path));
+
+ written.remove(path);
+ }
+
+ assertEquals("No more files were written to fs", 0, written.size());
+ }
+
+ // Compare two files as listed in the original namespace FileStatus and
+ // the output of the ls file from the image processor
+ private void compareFiles(FileStatus fs, LsElements elements) {
+ assertEquals("directory listed as such",
+ fs.isDir() ? 'd' : '-', elements.dir);
+ assertEquals("perms string equal",
+ fs.getPermission().toString(), elements.perms);
+ assertEquals("replication equal", fs.getReplication(), elements.replication);
+ assertEquals("owner equal", fs.getOwner(), elements.username);
+ assertEquals("group equal", fs.getGroup(), elements.groupname);
+ assertEquals("lengths equal", fs.getLen(), elements.filesize);
+ }
+
+ // Read the contents of the file created by the Ls processor
+ private HashMap<String, LsElements> readLsfile(File lsFile) throws IOException {
+ BufferedReader br = new BufferedReader(new FileReader(lsFile));
+ String line = null;
+ HashMap<String, LsElements> fileContents = new HashMap<String, LsElements>();
+
+ while((line = br.readLine()) != null)
+ readLsLine(line, fileContents);
+
+ return fileContents;
+ }
+
+ // Parse a line from the ls output. Store permissions, replication,
+ // username, groupname and filesize in hashmap keyed to the path name
+ private void readLsLine(String line, HashMap<String, LsElements> fileContents) {
+ String elements [] = line.split("\\s+");
+
+ assertEquals("Not enough elements in ls output", 8, elements.length);
+
+ LsElements lsLine = new LsElements();
+
+ lsLine.dir = elements[0].charAt(0);
+ lsLine.perms = elements[0].substring(1);
+ lsLine.replication = elements[1].equals("-")
+ ? 0 : Integer.valueOf(elements[1]);
+ lsLine.username = elements[2];
+ lsLine.groupname = elements[3];
+ lsLine.filesize = Long.valueOf(elements[4]);
+ // skipping date and time
+
+ String path = elements[7];
+
+ // Check that each file in the ls output was listed once
+ assertFalse("LS file had duplicate file entries",
+ fileContents.containsKey(path));
+
+ fileContents.put(path, lsLine);
+ }
+
+ // Copy one fsimage to another, changing the layout version in the process
+ private void changeLayoutVersion(File src, File dest, int newVersion)
+ throws IOException {
+ DataInputStream in = null;
+ DataOutputStream out = null;
+
+ try {
+ in = new DataInputStream(new FileInputStream(src));
+ out = new DataOutputStream(new FileOutputStream(dest));
+
+ in.readInt();
+ out.writeInt(newVersion);
+
+ byte [] b = new byte[1024];
+ while( in.read(b) > 0 ) {
+ out.write(b);
+ }
+ } finally {
+ if(in != null) in.close();
+ if(out != null) out.close();
+ }
+ }
+
+ // Only copy part of file into the other. Used for testing truncated fsimage
+ private void copyPartOfFile(File src, File dest) throws IOException {
+ InputStream in = null;
+ OutputStream out = null;
+
+ byte [] b = new byte[256];
+ int bytesWritten = 0;
+ int count;
+ int maxBytes = 700;
+
+ try {
+ in = new FileInputStream(src);
+ out = new FileOutputStream(dest);
+
+ while( (count = in.read(b)) > 0 && bytesWritten < maxBytes ) {
+ out.write(b);
+ bytesWritten += count;
+ }
+ } finally {
+ if(in != null) in.close();
+ if(out != null) out.close();
+ }
+ }
+
+ // Copy one file's contents into the other
+ private void copyFile(File src, File dest) throws IOException {
+ InputStream in = null;
+ OutputStream out = null;
+
+ try {
+ in = new FileInputStream(src);
+ out = new FileOutputStream(dest);
+
+ byte [] b = new byte[1024];
+ while( in.read(b) > 0 ) {
+ out.write(b);
+ }
+ } finally {
+ if(in != null) in.close();
+ if(out != null) out.close();
+ }
+ }
+}
Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18?rev=767331&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19?rev=767331&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream