You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2007/08/14 03:12:16 UTC
svn commit: r565595 - in /lucene/hadoop/branches/branch-0.14: ./
src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/
Author: nigel
Date: Mon Aug 13 18:12:15 2007
New Revision: 565595
URL: http://svn.apache.org/viewvc?view=rev&rev=565595
Log:
HADOOP-1629. Added a upgrade test for HADOOP-1134. Contributed by Raghu.
Added:
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java (with props)
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz (with props)
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt
Modified:
lucene/hadoop/branches/branch-0.14/CHANGES.txt
lucene/hadoop/branches/branch-0.14/build.xml
lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/Command.java
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
Modified: lucene/hadoop/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/CHANGES.txt?view=diff&rev=565595&r1=565594&r2=565595
==============================================================================
--- lucene/hadoop/branches/branch-0.14/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.14/CHANGES.txt Mon Aug 13 18:12:15 2007
@@ -511,6 +511,9 @@
151. HADOOP-1668. Added an INCOMPATIBILITY section to CHANGES.txt. (nigel)
+152. HADOOP-1629. Added a upgrade test for HADOOP-1134.
+ (Raghu Angadi via nigel)
+
Release 0.13.0 - 2007-06-08
1. HADOOP-1047. Fix TestReplication to succeed more reliably.
Modified: lucene/hadoop/branches/branch-0.14/build.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/build.xml?view=diff&rev=565595&r1=565594&r2=565595
==============================================================================
--- lucene/hadoop/branches/branch-0.14/build.xml (original)
+++ lucene/hadoop/branches/branch-0.14/build.xml Mon Aug 13 18:12:15 2007
@@ -448,6 +448,8 @@
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.txt" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.jar" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/mapred/test.zip" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt" todir="${test.cache.data}"/>
</target>
<!-- ================================================================== -->
@@ -463,6 +465,7 @@
fork="yes" maxmemory="256m" dir="${basedir}" timeout="${test.timeout}"
errorProperty="tests.failed" failureProperty="tests.failed">
<sysproperty key="test.build.data" value="${test.build.data}"/>
+ <sysproperty key="test.cache.data" value="${test.cache.data}"/>
<sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
<sysproperty key="test.src.dir" value="${test.src.dir}"/>
<sysproperty key="java.library.path"
Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/Command.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/Command.java?view=diff&rev=565595&r1=565594&r2=565595
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/Command.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/Command.java Mon Aug 13 18:12:15 2007
@@ -49,4 +49,44 @@
/** Parse the execution result */
protected abstract void parseExecResult(BufferedReader lines)
throws IOException;
+
+ /// A simple implementation of Command
+ private static class SimpleCommandExecutor extends Command {
+
+ private String[] command;
+ private StringBuffer reply;
+
+ SimpleCommandExecutor(String[] execString) {
+ command = execString;
+ }
+
+ @Override
+ protected String[] getExecString() {
+ return command;
+ }
+
+ @Override
+ protected void parseExecResult(BufferedReader lines) throws IOException {
+ reply = new StringBuffer();
+ char[] buf = new char[512];
+ int nRead;
+ while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
+ reply.append(buf, 0, nRead);
+ }
+ }
+
+ String getReply() {
+ return (reply == null) ? "" : reply.toString();
+ }
}
+
+ /**
+ * Static method to execute a command. Covers most of the simple cases
+ * without requiring the user to implement Command interface.
+ */
+ public static String execCommand(String[] cmd) throws IOException {
+ SimpleCommandExecutor exec = new SimpleCommandExecutor(cmd);
+ exec.run();
+ return exec.getReply();
+ }
+}
Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java?view=diff&rev=565595&r1=565594&r2=565595
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/MiniDFSCluster.java Mon Aug 13 18:12:15 2007
@@ -178,8 +178,9 @@
String[] args = (operation == null ||
operation == StartupOption.FORMAT ||
operation == StartupOption.REGULAR) ?
- new String[] {} : new String[] {"-"+operation.toString()};
-
+ null : new String[] {"-"+operation.toString()};
+ String [] dnArgs = (operation == StartupOption.UPGRADE) ? null : args;
+
for (int i = 0; i < numDataNodes; i++) {
Configuration dnConf = new Configuration(conf);
if (manageDfsDirs) {
@@ -198,7 +199,7 @@
}
System.out.println("Starting DataNode " + i + " with dfs.data.dir: "
+ dnConf.get("dfs.data.dir"));
- dataNodes.add(DataNode.createDataNode(args, dnConf));
+ dataNodes.add(DataNode.createDataNode(dnArgs, dnConf));
}
}
Added: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java?view=auto&rev=565595
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java (added)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java Mon Aug 13 18:12:15 2007
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import junit.framework.TestCase;
+import java.io.*;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.TreeMap;
+import java.util.zip.CRC32;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Command;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.dfs.FSConstants.StartupOption;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This tests data transfer protocol handling in the Datanode. It sends
+ * various forms of wrong data and verifies that Datanode handles it well.
+ *
+ * This test uses the following two file from src/test/.../dfs directory :
+ * 1) hadoop-12-dfs-dir.tgz : contains the tar of
+ * 2) hadoop-12-dfs-dir.txt : checksums that are compared in this test.
+ */
+public class TestDFSUpgradeFromImage extends TestCase {
+
+ private static final Log LOG = LogFactory.getLog(
+ "org.apache.hadoop.dfs.TestDFSUpgradeFromImage");
+
+ int numDataNodes = 4;
+
+ private static class ReferenceFileInfo {
+ String path;
+ long checksum;
+ }
+
+ LinkedList<ReferenceFileInfo> refList = new LinkedList<ReferenceFileInfo>();
+ Iterator<ReferenceFileInfo> refIter;
+
+ boolean printChecksum = false;
+
+ protected void setUp() throws IOException {
+ String tarFile = System.getProperty("test.cache.data") +
+ "/hadoop-12-dfs-dir.tgz";
+ String dataDir = System.getProperty("test.build.data");
+ File dfsDir = new File(dataDir, "dfs");
+ if ( dfsDir.exists() && !FileUtil.fullyDelete(dfsDir) ) {
+ throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
+ }
+
+ LOG.info("Unpacking the tar file " + tarFile);
+ String[] cmd = { "tar", "-zxf", tarFile, "-C", dataDir };
+ Command.execCommand(cmd);
+
+ //Now read the reference info
+
+ BufferedReader reader = new BufferedReader(
+ new FileReader(System.getProperty("test.cache.data") +
+ "/hadoop-12-dfs-dir.txt"));
+ String line;
+ while ( (line = reader.readLine()) != null ) {
+
+ line = line.trim();
+ if (line.length() <= 0 || line.startsWith("#")) {
+ continue;
+ }
+ String[] arr = line.split("\\s+\t\\s+");
+ if (arr.length < 1) {
+ continue;
+ }
+ if (arr[0].equals("printChecksums")) {
+ printChecksum = true;
+ break;
+ }
+ if (arr.length < 2) {
+ continue;
+ }
+ ReferenceFileInfo info = new ReferenceFileInfo();
+ info.path = arr[0];
+ info.checksum = Long.parseLong(arr[1]);
+ refList.add(info);
+ }
+ reader.close();
+ }
+
+ private void verifyChecksum(String path, long checksum) throws IOException {
+ if ( refIter == null ) {
+ refIter = refList.iterator();
+ }
+
+ if ( printChecksum ) {
+ LOG.info("CRC info for reference file : " + path + " \t " + checksum);
+ } else {
+ if ( !refIter.hasNext() ) {
+ throw new IOException("Checking checksum for " + path +
+ "Not enough elements in the refList");
+ }
+ ReferenceFileInfo info = refIter.next();
+ // The paths are expected to be listed in the same order
+ // as they are traversed here.
+ assertEquals(info.path, path);
+ assertEquals("Checking checksum for " + path, info.checksum, checksum);
+ }
+ }
+
+ CRC32 overallChecksum = new CRC32();
+
+ private void verifyDir(DFSClient client, String dir)
+ throws IOException {
+
+ DFSFileInfo[] fileArr = client.listPaths(new UTF8(dir));
+ TreeMap<String, Boolean> fileMap = new TreeMap<String, Boolean>();
+
+ for(DFSFileInfo file : fileArr) {
+ String path = file.getPath();
+ fileMap.put(path, Boolean.valueOf(file.isDir()));
+ }
+
+ for(Iterator<String> it = fileMap.keySet().iterator(); it.hasNext();) {
+ String path = it.next();
+ boolean isDir = fileMap.get(path);
+
+ overallChecksum.update(path.getBytes());
+
+ if ( isDir ) {
+ verifyDir(client, path);
+ } else {
+ // this is not a directory. Checksum the file data.
+ CRC32 fileCRC = new CRC32();
+ FSInputStream in = client.open(new UTF8(path));
+ byte[] buf = new byte[4096];
+ int nRead = 0;
+ while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
+ fileCRC.update(buf, 0, nRead);
+ }
+
+ verifyChecksum(path, fileCRC.getValue());
+ }
+ }
+ }
+
+ private void verifyFileSystem(DFSClient client) throws IOException {
+
+ verifyDir(client, "/");
+
+ verifyChecksum("overallCRC", overallChecksum.getValue());
+
+ if ( printChecksum ) {
+ throw new IOException("Checksums are written to log as requested. " +
+ "Throwing this exception to force an error " +
+ "for this test.");
+ }
+ }
+
+ public void testUpgradeFromImage() throws IOException {
+
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster(0, conf, numDataNodes, false,
+ true, StartupOption.UPGRADE,
+ null);
+ cluster.waitActive();
+ DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
+ cluster.getNameNodePort()),
+ conf);
+ //Safemode will be off only after upgrade is complete. Wait for it.
+ while ( dfsClient.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET) ) {
+ LOG.info("Waiting for SafeMode to be OFF.");
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {}
+ }
+
+ verifyFileSystem(dfsClient);
+ }
+}
Propchange: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz?view=auto&rev=565595
==============================================================================
Binary file - no diff available.
Propchange: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.tgz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt?view=auto&rev=565595
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt (added)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/hadoop-12-dfs-dir.txt Mon Aug 13 18:12:15 2007
@@ -0,0 +1,69 @@
+#
+# This is a readme for hadoop-12-dir.tgz and hadoop-12-dir.txt.
+#
+# 08/08/2007:
+#
+# See HADOOP-1629 for more info if needed.
+# These two files are used by unit test TestDFSUpgradeFromImage.java
+#
+# hadoop-12-dfs-dir.tgz :
+# ---------------------
+# This file contains the HDFS directory structure for one namenode and 4 datanodes.
+# The structure is setup similar to the structure used in MiniDFSCluster.
+# The directory was created with Hadoo-0.12.x (svn revision 526216).
+#
+# In the test, this directory is unpacked and MiniDFSCluster is run with
+# "-upgrade" option. The test waits for the upgrade to complete
+# (leave safe mode) and then all the files are read. The test checks that the
+# directory structure and file checksums exactly match the information
+# in this file.
+#
+# hadoop-12-dfs-dir.txt :
+# ---------------------
+# Along with this description this file contains the expected files and
+# checksums or the files in the upgraded DFS.
+#
+# The original DFS directory was created with various types of files and with
+# some recoverable errors (i.e. corrupt or missing .crc files).
+#
+# A similar set of files exist in two different DFS directories.
+# For e.g. "top-dir-1Mb-12" contains files created with dfs.block.size of 1Mb
+# and io.bytes.per.checksum of 512.
+#
+# In the future, when Hadoop project no longer supports upgrade from
+# Hadoop-0.12, then a new DFS directory image must be created.
+#
+# To generate checksum info for new files :
+# ---------------------------------------
+# Uncomment the last coment (starts with "printChecksums") and run the
+# test again. When the test sees this line, it prints the checksum
+# information that should replace the checksum information in
+# this file. When run in this mode, the test will fail with a descriptive IOException.
+#
+# Next, extract the checksum info from the test log like this:
+# sed -n 's/.*CRC info for reference file : //p' test-log.txt >> this_file
+# This will append a new list of files and checksums to this file. Be sure to remove the existing checksum info.
+#
+# For your reference, the format of the checksum info below is "filename whitespace*\twhitespace* checksum\n"
+#
+# Uncomment the following line to produce checksum info for a new DFS image.
+#printChecksums
+
+/1kb-multiple-checksum-blocks-64-16 191893480
+/top-dir-120000-60/1Mb-file 4079112547
+/top-dir-120000-60/4k-file 3716287280
+/top-dir-120000-60/5Mb-file 2563834633
+/top-dir-120000-60/directory1/500thousand-file 3036538664
+/top-dir-120000-60/directory1/file-with-corrupt-crc 1984689737
+/top-dir-120000-60/directory1/file-with-no-crc 4004594475
+/top-dir-120000-60/directory1/zero1 0
+/top-dir-120000-60/zerolen 0
+/top-dir-1Mb-512/1Mb-file 4079112547
+/top-dir-1Mb-512/4k-file 3716287280
+/top-dir-1Mb-512/5Mb-file 2563834633
+/top-dir-1Mb-512/directory1/500thousand-file 3036538664
+/top-dir-1Mb-512/directory1/file-with-corrupt-crc 1984689737
+/top-dir-1Mb-512/directory1/file-with-no-crc 4004594475
+/top-dir-1Mb-512/directory1/zero1 0
+/top-dir-1Mb-512/zerolen 0
+overallCRC 1419480698