You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2011/10/17 22:42:33 UTC
svn commit: r1185363 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: jitendra
Date: Mon Oct 17 20:42:32 2011
New Revision: 1185363
URL: http://svn.apache.org/viewvc?rev=1185363&view=rev
Log:
Merged r1185354 from trunk for HDFS-2188.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1185363&r1=1185362&r2=1185363&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Oct 17 20:42:32 2011
@@ -97,6 +97,9 @@ Trunk (unreleased changes)
HDFS-2298. Fix TestDfsOverAvroRpc by changing ClientProtocol to
not include multiple methods of the same name. (cutting)
+ HDFS-2188. Make FSEditLog create its journals from a list of URIs rather
+ than NNStorage. (Ivan Kelly via jitendra)
+
Release 0.23.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1185363&r1=1185362&r2=1185363&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Mon Oct 17 20:42:32 2011
@@ -18,10 +18,11 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.now;
-
+import java.net.URI;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -42,9 +43,11 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.conf.Configuration;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
/**
* FSEditLog maintains a log of the namespace modifications.
@@ -122,23 +125,68 @@ public class FSEditLog {
}
};
+ final private Collection<URI> editsDirs;
+
+ /**
+ * Construct FSEditLog with default configuration, taking editDirs from NNStorage
+ * @param storage Storage object used by namenode
+ */
+ @VisibleForTesting
FSEditLog(NNStorage storage) {
+ this(new Configuration(), storage, Collections.<URI>emptyList());
+ }
+
+ /**
+ * Constructor for FSEditLog. Add underlying journals are constructed, but
+ * no streams are opened until open() is called.
+ *
+ * @param conf The namenode configuration
+ * @param storage Storage object used by namenode
+ * @param editsDirs List of journals to use
+ */
+ FSEditLog(Configuration conf, NNStorage storage, Collection<URI> editsDirs) {
isSyncRunning = false;
this.storage = storage;
metrics = NameNode.getNameNodeMetrics();
lastPrintTime = now();
+
+ if (editsDirs.isEmpty()) {
+ // if this is the case, no edit dirs have been explictly configured
+ // image dirs are to be used for edits too
+ try {
+ editsDirs = Lists.newArrayList(storage.getEditsDirectories());
+ } catch (IOException ioe) {
+ // cannot get list from storage, so the empty editsDirs
+ // will be assigned. an error will be thrown on first use
+ // of the editlog, as no journals will exist
+ }
+ this.editsDirs = editsDirs;
+ } else {
+ this.editsDirs = Lists.newArrayList(editsDirs);
+ }
this.journalSet = new JournalSet();
- for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
- journalSet.add(new FileJournalManager(sd));
+ for (URI u : this.editsDirs) {
+ StorageDirectory sd = storage.getStorageDirectory(u);
+ if (sd != null) {
+ journalSet.add(new FileJournalManager(sd));
+ }
}
-
+
if (journalSet.isEmpty()) {
LOG.error("No edits directories configured!");
}
state = State.BETWEEN_LOG_SEGMENTS;
}
-
+
+ /**
+ * Get the list of URIs the editlog is using for storage
+ * @return collection of URIs in use by the edit log
+ */
+ Collection<URI> getEditURIs() {
+ return editsDirs;
+ }
+
/**
* Initialize the output stream for logging, opening the first
* log segment.
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1185363&r1=1185362&r2=1185363&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Mon Oct 17 20:42:32 2011
@@ -120,7 +120,7 @@ public class FSImage implements Closeabl
storage.setRestoreFailedStorage(true);
}
- this.editLog = new FSEditLog(storage);
+ this.editLog = new FSEditLog(conf, storage, editsDirs);
archivalManager = new NNStorageRetentionManager(conf, storage, editLog);
}
@@ -150,8 +150,7 @@ public class FSImage implements Closeabl
"NameNode formatting should be performed before reading the image";
Collection<URI> imageDirs = storage.getImageDirectories();
- Collection<URI> editsDirs = storage.getEditsDirectories();
-
+ Collection<URI> editsDirs = editLog.getEditURIs();
// none of the data dirs exist
if((imageDirs.size() == 0 || editsDirs.size() == 0)
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1185363&r1=1185362&r2=1185363&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Mon Oct 17 20:42:32 2011
@@ -59,6 +59,7 @@ import org.apache.hadoop.net.DNS;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
/**
* NNStorage is responsible for management of the StorageDirectories used by
@@ -154,7 +155,9 @@ public class NNStorage extends Storage i
storageDirs = new CopyOnWriteArrayList<StorageDirectory>();
- setStorageDirectories(imageDirs, editsDirs);
+ // this may modify the editsDirs, so copy before passing in
+ setStorageDirectories(imageDirs,
+ Lists.newArrayList(editsDirs));
}
@Override // Storage
@@ -299,6 +302,27 @@ public class NNStorage extends Storage i
}
/**
+ * Return the storage directory corresponding to the passed URI
+ * @param uri URI of a storage directory
+ * @return The matching storage directory or null if none found
+ */
+ StorageDirectory getStorageDirectory(URI uri) {
+ try {
+ uri = Util.fileAsURI(new File(uri));
+ Iterator<StorageDirectory> it = dirIterator();
+ for (; it.hasNext(); ) {
+ StorageDirectory sd = it.next();
+ if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
+ return sd;
+ }
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Error converting file to URI", ioe);
+ }
+ return null;
+ }
+
+ /**
* Checks the consistency of a URI, in particular if the scheme
* is specified and is supported by a concrete implementation
* @param u URI whose consistency is being checked.
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1185363&r1=1185362&r2=1185363&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Mon Oct 17 20:42:32 2011
@@ -41,10 +41,13 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
import org.mockito.Mockito;
+import org.mockito.Matchers;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
@@ -105,7 +108,7 @@ public abstract class FSImageTestUtil {
Mockito.doReturn(type)
.when(sd).getStorageDirType();
Mockito.doReturn(currentDir).when(sd).getCurrentDir();
-
+ Mockito.doReturn(currentDir).when(sd).getRoot();
Mockito.doReturn(mockFile(true)).when(sd).getVersionFile();
Mockito.doReturn(mockFile(false)).when(sd).getPreviousDir();
return sd;
@@ -127,7 +130,8 @@ public abstract class FSImageTestUtil {
// Version file should always exist
doReturn(mockFile(true)).when(sd).getVersionFile();
-
+ doReturn(mockFile(true)).when(sd).getRoot();
+
// Previous dir optionally exists
doReturn(mockFile(previousExists))
.when(sd).getPreviousDir();
@@ -142,6 +146,7 @@ public abstract class FSImageTestUtil {
doReturn(files).when(mockDir).listFiles();
doReturn(mockDir).when(sd).getCurrentDir();
+
return sd;
}
@@ -169,11 +174,16 @@ public abstract class FSImageTestUtil {
assertTrue(logDir.mkdirs() || logDir.exists());
Files.deleteDirectoryContents(logDir);
NNStorage storage = Mockito.mock(NNStorage.class);
- List<StorageDirectory> sds = Lists.newArrayList(
- FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS));
+ StorageDirectory sd
+ = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS);
+ List<StorageDirectory> sds = Lists.newArrayList(sd);
Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS);
+ Mockito.doReturn(sd).when(storage)
+ .getStorageDirectory(Matchers.<URI>anyObject());
- return new FSEditLog(storage);
+ return new FSEditLog(new Configuration(),
+ storage,
+ ImmutableList.of(logDir.toURI()));
}
/**
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1185363&r1=1185362&r2=1185363&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Mon Oct 17 20:42:32 2011
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -861,8 +862,11 @@ public class TestEditLog extends TestCas
* The syntax <code>[1,]</code> specifies an in-progress log starting at
* txid 1.
*/
- private NNStorage mockStorageWithEdits(String... editsDirSpecs) {
+ private NNStorage mockStorageWithEdits(String... editsDirSpecs) throws IOException {
List<StorageDirectory> sds = Lists.newArrayList();
+ List<URI> uris = Lists.newArrayList();
+
+ NNStorage storage = Mockito.mock(NNStorage.class);
for (String dirSpec : editsDirSpecs) {
List<String> files = Lists.newArrayList();
String[] logSpecs = dirSpec.split("\\|");
@@ -878,13 +882,17 @@ public class TestEditLog extends TestCas
Long.valueOf(m.group(2))));
}
}
- sds.add(FSImageTestUtil.mockStorageDirectory(
+ StorageDirectory sd = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.EDITS, false,
- files.toArray(new String[0])));
- }
-
- NNStorage storage = Mockito.mock(NNStorage.class);
+ files.toArray(new String[0]));
+ sds.add(sd);
+ URI u = URI.create("file:///storage"+ Math.random());
+ Mockito.doReturn(sd).when(storage).getStorageDirectory(u);
+ uris.add(u);
+ }
+
Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS);
+ Mockito.doReturn(uris).when(storage).getEditsDirectories();
return storage;
}