You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/02 01:59:20 UTC
hadoop git commit: HDFS-7929. inotify unable fetch pre-upgrade edit
log segments once upgrade starts (Zhe Zhang via Colin P. McCabe)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 49ef26b5a -> 81a445edf
HDFS-7929. inotify unable fetch pre-upgrade edit log segments once upgrade starts (Zhe Zhang via Colin P. McCabe)
(cherry picked from commit 43b41f22411439c5e23629197fb2fde45dcf0f0f)
(cherry picked from commit 219eb22c1571f76df32967a930049d983cbf5024)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
(cherry picked from commit 03798416bfe27383c52e4d9f632fe9fa168c6e95)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81a445ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81a445ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81a445ed
Branch: refs/heads/branch-2.6.1
Commit: 81a445edf81f42c90a05d764dfebfadfafad622b
Parents: 49ef26b
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Mar 18 18:48:54 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 1 16:55:50 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/namenode/FSImage.java | 2 +-
.../server/namenode/FileJournalManager.java | 2 +-
.../hdfs/server/namenode/NNUpgradeUtil.java | 44 ++++++++++++++++--
.../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 48 +++++++++++++++++++-
5 files changed, 93 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 44a7139..ff335ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -97,6 +97,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7587. Edit log corruption can happen if append fails with a quota
violation. (jing9)
+ HDFS-7929. inotify unable fetch pre-upgrade edit log segments once upgrade
+ starts (Zhe Zhang via Colin P. McCabe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 9b72421..51efb51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -393,7 +393,7 @@ public class FSImage implements Closeable {
for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext();) {
StorageDirectory sd = it.next();
try {
- NNUpgradeUtil.doPreUpgrade(sd);
+ NNUpgradeUtil.doPreUpgrade(conf, sd);
} catch (Exception e) {
LOG.error("Failed to move aside pre-upgrade storage " +
"in image directory " + sd.getRoot(), e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
index 101c42c..2df052b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
@@ -585,7 +585,7 @@ public class FileJournalManager implements JournalManager {
public void doPreUpgrade() throws IOException {
LOG.info("Starting upgrade of edits directory " + sd.getRoot());
try {
- NNUpgradeUtil.doPreUpgrade(sd);
+ NNUpgradeUtil.doPreUpgrade(conf, sd);
} catch (IOException ioe) {
LOG.error("Failed to move aside pre-upgrade storage " +
"in image directory " + sd.getRoot(), ioe);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
index 546480d..c63da20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
@@ -18,10 +18,13 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
@@ -99,15 +102,17 @@ abstract class NNUpgradeUtil {
* a call to any JM's or local storage dir's doPreUpgrade method fails, then
* doUpgrade will not be called for any JM. The existing current dir is
* renamed to previous.tmp, and then a new, empty current dir is created.
- *
+ *
+ * @param conf configuration for creating {@link EditLogFileOutputStream}
* @param sd the storage directory to perform the pre-upgrade procedure.
* @throws IOException in the event of error
*/
- static void doPreUpgrade(StorageDirectory sd) throws IOException {
+ static void doPreUpgrade(Configuration conf, StorageDirectory sd)
+ throws IOException {
LOG.info("Starting upgrade of storage directory " + sd.getRoot());
File curDir = sd.getCurrentDir();
File prevDir = sd.getPreviousDir();
- File tmpDir = sd.getPreviousTmp();
+ final File tmpDir = sd.getPreviousTmp();
Preconditions.checkState(curDir.exists(),
"Current directory must exist for preupgrade.");
@@ -123,6 +128,39 @@ abstract class NNUpgradeUtil {
if (!curDir.mkdir()) {
throw new IOException("Cannot create directory " + curDir);
}
+
+ String[] fileNameList = tmpDir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return dir.equals(tmpDir)
+ && name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
+ }
+ });
+ if (fileNameList == null) {
+ throw new IOException("either " + tmpDir +
+ " is not a directory or there is an I/O error");
+ }
+
+ for (String s : fileNameList) {
+ File prevFile = new File(tmpDir, s);
+ Preconditions.checkState(prevFile.canRead(),
+ "Edits log file " + s + " is not readable.");
+ File newFile = new File(curDir, prevFile.getName());
+ Preconditions.checkState(newFile.createNewFile(),
+ "Cannot create new edits log file in " + curDir);
+ EditLogFileInputStream in = new EditLogFileInputStream(prevFile);
+ EditLogFileOutputStream out =
+ new EditLogFileOutputStream(conf, newFile, 512*1024);
+ FSEditLogOp logOp = in.nextValidOp();
+ while (logOp != null) {
+ out.write(logOp);
+ logOp = in.nextOp();
+ }
+ out.setReadyToFlush();
+ out.flushAndSync(true);
+ out.close();
+ in.close();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
index bb00144..49762ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
@@ -28,7 +28,11 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
@@ -42,6 +46,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
@@ -450,7 +455,48 @@ public class TestDFSUpgrade {
assertTrue(Storage.is203LayoutVersion(lv));
}
}
-
+
+ @Test
+ public void testPreserveEditLogs() throws Exception {
+ conf = new HdfsConfiguration();
+ conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
+ String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
+ conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
+
+ log("Normal NameNode upgrade", 1);
+ File[] created =
+ UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
+ List<String> beforeUpgrade = new LinkedList<String>();
+ for (final File createdDir : created) {
+ String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE);
+ Collections.addAll(beforeUpgrade, fileNameList);
+ }
+
+ cluster = createCluster();
+
+ List<String> afterUpgrade = new LinkedList<String>();
+ for (final File createdDir : created) {
+ String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE);
+ Collections.addAll(afterUpgrade, fileNameList);
+ }
+
+ for (String s : beforeUpgrade) {
+ assertTrue(afterUpgrade.contains(s));
+ }
+
+ cluster.shutdown();
+ UpgradeUtilities.createEmptyDirs(nameNodeDirs);
+ }
+
+ private static enum EditLogsFilter implements FilenameFilter {
+ INSTANCE;
+
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
+ }
+ }
+
public static void main(String[] args) throws Exception {
TestDFSUpgrade t = new TestDFSUpgrade();
TestDFSUpgrade.initialize();