You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/10/19 04:28:07 UTC
svn commit: r1399950 [14/27] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project: ./
hadoop-hdfs-httpfs/ hadoop-hdfs-httpfs/dev-support/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apac...
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java Fri Oct 19 02:25:55 2012
@@ -26,6 +26,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.PriorityQueue;
import java.util.SortedSet;
+import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,6 +34,7 @@ import org.apache.hadoop.classification.
import static org.apache.hadoop.util.ExitUtil.terminate;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -146,7 +148,7 @@ public class JournalSet implements Journ
return journal;
}
- private boolean isDisabled() {
+ boolean isDisabled() {
return disabled;
}
@@ -164,8 +166,12 @@ public class JournalSet implements Journ
return required;
}
}
-
- private List<JournalAndStream> journals = Lists.newArrayList();
+
+ // COW implementation is necessary since some users (eg the web ui) call
+ // getAllJournalStreams() and then iterate. Since this is rarely
+ // mutated, there is no performance concern.
+ private List<JournalAndStream> journals =
+ new CopyOnWriteArrayList<JournalSet.JournalAndStream>();
final int minimumRedundantJournals;
JournalSet(int minimumRedundantResources) {
@@ -173,6 +179,20 @@ public class JournalSet implements Journ
}
@Override
+ public void format(NamespaceInfo nsInfo) throws IOException {
+ // The iteration is done by FSEditLog itself
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasSomeData() throws IOException {
+ // This is called individually on the underlying journals,
+ // not on the JournalSet.
+ throw new UnsupportedOperationException();
+ }
+
+
+ @Override
public EditLogOutputStream startLogSegment(final long txId) throws IOException {
mapJournalsAndReportErrors(new JournalClosure() {
@Override
@@ -227,8 +247,20 @@ public class JournalSet implements Journ
LOG.info("Skipping jas " + jas + " since it's disabled");
continue;
}
- jas.getManager().selectInputStreams(allStreams, fromTxId, inProgressOk);
+ try {
+ jas.getManager().selectInputStreams(allStreams, fromTxId, inProgressOk);
+ } catch (IOException ioe) {
+ LOG.warn("Unable to determine input streams from " + jas.getManager() +
+ ". Skipping.", ioe);
+ }
}
+ chainAndMakeRedundantStreams(streams, allStreams, fromTxId, inProgressOk);
+ }
+
+ public static void chainAndMakeRedundantStreams(
+ Collection<EditLogInputStream> outStreams,
+ PriorityQueue<EditLogInputStream> allStreams,
+ long fromTxId, boolean inProgressOk) {
// We want to group together all the streams that start on the same start
// transaction ID. To do this, we maintain an accumulator (acc) of all
// the streams we've seen at a given start transaction ID. When we see a
@@ -246,7 +278,7 @@ public class JournalSet implements Journ
if (accFirstTxId == elis.getFirstTxId()) {
acc.add(elis);
} else if (accFirstTxId < elis.getFirstTxId()) {
- streams.add(new RedundantEditLogInputStream(acc, fromTxId));
+ outStreams.add(new RedundantEditLogInputStream(acc, fromTxId));
acc.clear();
acc.add(elis);
} else if (accFirstTxId > elis.getFirstTxId()) {
@@ -257,7 +289,7 @@ public class JournalSet implements Journ
}
}
if (!acc.isEmpty()) {
- streams.add(new RedundantEditLogInputStream(acc, fromTxId));
+ outStreams.add(new RedundantEditLogInputStream(acc, fromTxId));
acc.clear();
}
}
@@ -439,12 +471,12 @@ public class JournalSet implements Journ
}
@Override
- protected void flushAndSync() throws IOException {
+ protected void flushAndSync(final boolean durable) throws IOException {
mapJournalsAndReportErrors(new JournalClosure() {
@Override
public void apply(JournalAndStream jas) throws IOException {
if (jas.isActive()) {
- jas.getCurrentStream().flushAndSync();
+ jas.getCurrentStream().flushAndSync(durable);
}
}
}, "flushAndSync");
@@ -497,7 +529,6 @@ public class JournalSet implements Journ
}
}
- @VisibleForTesting
List<JournalAndStream> getAllJournalStreams() {
return journals;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Fri Oct 19 02:25:55 2012
@@ -39,7 +39,7 @@ import org.apache.hadoop.util.Daemon;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import static org.apache.hadoop.hdfs.server.common.Util.now;
+import static org.apache.hadoop.util.Time.now;
/**
* LeaseManager does the lease housekeeping for writing on files.
@@ -159,6 +159,12 @@ public class LeaseManager {
}
}
+ synchronized void removeAllLeases() {
+ sortedLeases.clear();
+ sortedLeasesByPath.clear();
+ leases.clear();
+ }
+
/**
* Reassign lease for file src to the new holder.
*/
@@ -390,6 +396,7 @@ public class LeaseManager {
final String name = getClass().getSimpleName();
/** Check leases periodically. */
+ @Override
public void run() {
for(; shouldRunMonitor && fsnamesystem.isRunning(); ) {
try {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java Fri Oct 19 02:25:55 2012
@@ -54,6 +54,7 @@ public class ListPathsServlet extends Df
public static final ThreadLocal<SimpleDateFormat> df =
new ThreadLocal<SimpleDateFormat>() {
+ @Override
protected SimpleDateFormat initialValue() {
return HftpFileSystem.getDateFormat();
}
@@ -128,6 +129,7 @@ public class ListPathsServlet extends Df
* </listing>
* }
*/
+ @Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
final PrintWriter out = response.getWriter();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Fri Oct 19 02:25:55 2012
@@ -28,13 +28,10 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -46,18 +43,17 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
-import org.apache.hadoop.hdfs.server.common.UpgradeManager;
import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.util.PersistentLongFile;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
/**
* NNStorage is responsible for management of the StorageDirectories used by
@@ -66,8 +62,6 @@ import com.google.common.collect.Maps;
@InterfaceAudience.Private
public class NNStorage extends Storage implements Closeable,
StorageErrorReporter {
- private static final Log LOG = LogFactory.getLog(NNStorage.class.getName());
-
static final String DEPRECATED_MESSAGE_DIGEST_PROPERTY = "imageMD5Digest";
static final String LOCAL_URI_SCHEME = "file";
@@ -100,10 +94,12 @@ public class NNStorage extends Storage i
EDITS,
IMAGE_AND_EDITS;
+ @Override
public StorageDirType getStorageDirType() {
return this;
}
+ @Override
public boolean isOfType(StorageDirType type) {
if ((this == IMAGE_AND_EDITS) && (type == IMAGE || type == EDITS))
return true;
@@ -111,7 +107,6 @@ public class NNStorage extends Storage i
}
}
- private UpgradeManager upgradeManager = null;
protected String blockpoolID = ""; // id of the block pool
/**
@@ -550,11 +545,8 @@ public class NNStorage extends Storage i
public static NamespaceInfo newNamespaceInfo()
throws UnknownHostException {
- return new NamespaceInfo(
- newNamespaceID(),
- newClusterID(),
- newBlockPoolID(),
- 0L, 0);
+ return new NamespaceInfo(newNamespaceID(), newClusterID(),
+ newBlockPoolID(), 0L);
}
public void format() throws IOException {
@@ -599,13 +591,6 @@ public class NNStorage extends Storage i
String sbpid = props.getProperty("blockpoolID");
setBlockPoolID(sd.getRoot(), sbpid);
}
-
- String sDUS, sDUV;
- sDUS = props.getProperty("distributedUpgradeState");
- sDUV = props.getProperty("distributedUpgradeVersion");
- setDistributedUpgradeState(
- sDUS == null? false : Boolean.parseBoolean(sDUS),
- sDUV == null? getLayoutVersion() : Integer.parseInt(sDUV));
setDeprecatedPropertiesForUpgrade(props);
}
@@ -652,13 +637,6 @@ public class NNStorage extends Storage i
if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
props.setProperty("blockpoolID", blockpoolID);
}
- boolean uState = getDistributedUpgradeState();
- int uVersion = getDistributedUpgradeVersion();
- if(uState && uVersion != getLayoutVersion()) {
- props.setProperty("distributedUpgradeState", Boolean.toString(uState));
- props.setProperty("distributedUpgradeVersion",
- Integer.toString(uVersion));
- }
}
static File getStorageFile(StorageDirectory sd, NameNodeFile type, long imageTxId) {
@@ -731,7 +709,7 @@ public class NNStorage extends Storage i
* Return the first readable image file for the given txid, or null
* if no such image can be found
*/
- File findImageFile(long txid) throws IOException {
+ File findImageFile(long txid) {
return findFile(NameNodeDirType.IMAGE,
getImageFileName(txid));
}
@@ -753,76 +731,6 @@ public class NNStorage extends Storage i
}
/**
- * Set the upgrade manager for use in a distributed upgrade.
- * @param um The upgrade manager
- */
- void setUpgradeManager(UpgradeManager um) {
- upgradeManager = um;
- }
-
- /**
- * @return The current distribued upgrade state.
- */
- boolean getDistributedUpgradeState() {
- return upgradeManager == null ? false : upgradeManager.getUpgradeState();
- }
-
- /**
- * @return The current upgrade version.
- */
- int getDistributedUpgradeVersion() {
- return upgradeManager == null ? 0 : upgradeManager.getUpgradeVersion();
- }
-
- /**
- * Set the upgrade state and version.
- * @param uState the new state.
- * @param uVersion the new version.
- */
- private void setDistributedUpgradeState(boolean uState, int uVersion) {
- if (upgradeManager != null) {
- upgradeManager.setUpgradeState(uState, uVersion);
- }
- }
-
- /**
- * Verify that the distributed upgrade state is valid.
- * @param startOpt the option the namenode was started with.
- */
- void verifyDistributedUpgradeProgress(StartupOption startOpt
- ) throws IOException {
- if(startOpt == StartupOption.ROLLBACK || startOpt == StartupOption.IMPORT)
- return;
-
- assert upgradeManager != null : "FSNameSystem.upgradeManager is null.";
- if(startOpt != StartupOption.UPGRADE) {
- if(upgradeManager.getUpgradeState())
- throw new IOException(
- "\n Previous distributed upgrade was not completed. "
- + "\n Please restart NameNode with -upgrade option.");
- if(upgradeManager.getDistributedUpgrades() != null)
- throw new IOException("\n Distributed upgrade for NameNode version "
- + upgradeManager.getUpgradeVersion()
- + " to current LV " + HdfsConstants.LAYOUT_VERSION
- + " is required.\n Please restart NameNode"
- + " with -upgrade option.");
- }
- }
-
- /**
- * Initialize a distributed upgrade.
- */
- void initializeDistributedUpgrade() throws IOException {
- if(! upgradeManager.initializeUpgrade())
- return;
- // write new upgrade state into disk
- writeAll();
- LOG.info("\n Distributed upgrade for NameNode version "
- + upgradeManager.getUpgradeVersion() + " to current LV "
- + HdfsConstants.LAYOUT_VERSION + " is initialized.");
- }
-
- /**
* Disable the check for pre-upgradable layouts. Needed for BackupImage.
* @param val Whether to disable the preupgradeable layout check.
*/
@@ -996,7 +904,7 @@ public class NNStorage extends Storage i
}
int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
- String bpid = "BP-" + rand + "-"+ ip + "-" + System.currentTimeMillis();
+ String bpid = "BP-" + rand + "-"+ ip + "-" + Time.now();
return bpid;
}
@@ -1098,7 +1006,6 @@ public class NNStorage extends Storage i
getNamespaceID(),
getClusterID(),
getBlockPoolID(),
- getCTime(),
- getDistributedUpgradeVersion());
+ getCTime());
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java Fri Oct 19 02:25:55 2012
@@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.server.na
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.TreeSet;
@@ -32,6 +34,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -48,16 +51,17 @@ public class NNStorageRetentionManager {
private final int numCheckpointsToRetain;
private final long numExtraEditsToRetain;
+ private final int maxExtraEditsSegmentsToRetain;
private static final Log LOG = LogFactory.getLog(
NNStorageRetentionManager.class);
private final NNStorage storage;
private final StoragePurger purger;
- private final FSEditLog editLog;
+ private final LogsPurgeable purgeableLogs;
public NNStorageRetentionManager(
Configuration conf,
NNStorage storage,
- FSEditLog editLog,
+ LogsPurgeable purgeableLogs,
StoragePurger purger) {
this.numCheckpointsToRetain = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY,
@@ -65,6 +69,9 @@ public class NNStorageRetentionManager {
this.numExtraEditsToRetain = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_DEFAULT);
+ this.maxExtraEditsSegmentsToRetain = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_DEFAULT);
Preconditions.checkArgument(numCheckpointsToRetain > 0,
"Must retain at least one checkpoint");
Preconditions.checkArgument(numExtraEditsToRetain >= 0,
@@ -72,13 +79,13 @@ public class NNStorageRetentionManager {
" must not be negative");
this.storage = storage;
- this.editLog = editLog;
+ this.purgeableLogs = purgeableLogs;
this.purger = purger;
}
public NNStorageRetentionManager(Configuration conf, NNStorage storage,
- FSEditLog editLog) {
- this(conf, storage, editLog, new DeletionStoragePurger());
+ LogsPurgeable purgeableLogs) {
+ this(conf, storage, purgeableLogs, new DeletionStoragePurger());
}
public void purgeOldStorage() throws IOException {
@@ -94,8 +101,40 @@ public class NNStorageRetentionManager {
// provide a "cushion" of older txns that we keep, which is
// handy for HA, where a remote node may not have as many
// new images.
- long purgeLogsFrom = Math.max(0, minImageTxId + 1 - numExtraEditsToRetain);
- editLog.purgeLogsOlderThan(purgeLogsFrom);
+ //
+ // First, determine the target number of extra transactions to retain based
+ // on the configured amount.
+ long minimumRequiredTxId = minImageTxId + 1;
+ long purgeLogsFrom = Math.max(0, minimumRequiredTxId - numExtraEditsToRetain);
+
+ ArrayList<EditLogInputStream> editLogs = new ArrayList<EditLogInputStream>();
+ purgeableLogs.selectInputStreams(editLogs, purgeLogsFrom, false);
+ Collections.sort(editLogs, new Comparator<EditLogInputStream>() {
+ @Override
+ public int compare(EditLogInputStream a, EditLogInputStream b) {
+ return ComparisonChain.start()
+ .compare(a.getFirstTxId(), b.getFirstTxId())
+ .compare(a.getLastTxId(), b.getLastTxId())
+ .result();
+ }
+ });
+
+ // Next, adjust the number of transactions to retain if doing so would mean
+ // keeping too many segments around.
+ while (editLogs.size() > maxExtraEditsSegmentsToRetain) {
+ purgeLogsFrom = editLogs.get(0).getFirstTxId();
+ editLogs.remove(0);
+ }
+
+ // Finally, ensure that we're not trying to purge any transactions that we
+ // actually need.
+ if (purgeLogsFrom > minimumRequiredTxId) {
+ throw new AssertionError("Should not purge more edits than required to "
+ + "restore: " + purgeLogsFrom + " should be <= "
+ + minimumRequiredTxId);
+ }
+
+ purgeableLogs.purgeLogsOlderThan(purgeLogsFrom);
}
private void purgeCheckpointsOlderThan(
@@ -103,7 +142,6 @@ public class NNStorageRetentionManager {
long minTxId) {
for (FSImageFile image : inspector.getFoundImages()) {
if (image.getCheckpointTxId() < minTxId) {
- LOG.info("Purging old image " + image);
purger.purgeImage(image);
}
}
@@ -146,11 +184,13 @@ public class NNStorageRetentionManager {
static class DeletionStoragePurger implements StoragePurger {
@Override
public void purgeLog(EditLogFile log) {
+ LOG.info("Purging old edit log " + log);
deleteOrWarn(log.getFile());
}
@Override
public void purgeImage(FSImageFile image) {
+ LOG.info("Purging old image " + image);
deleteOrWarn(image.getFile());
deleteOrWarn(MD5FileUtils.getDigestFileForFile(image.getFile()));
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Oct 19 02:25:55 2012
@@ -17,18 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,12 +36,13 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.ha.ServiceFailedException;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Trash;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.util.ExitUtil.terminate;
+import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -53,9 +50,6 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
-import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -67,8 +61,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
-import org.apache.hadoop.hdfs.util.AtomicFileOutputStream;
-import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -79,14 +72,13 @@ import org.apache.hadoop.security.Securi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
+import org.apache.hadoop.util.ExitUtil.ExitException;
+import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.ExitUtil.ExitException;
-
-import static org.apache.hadoop.util.ExitUtil.terminate;
-import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
/**********************************************************
@@ -197,6 +189,22 @@ public class NameNode {
DFS_HA_AUTO_FAILOVER_ENABLED_KEY
};
+ private static final String USAGE = "Usage: java NameNode ["
+ + StartupOption.BACKUP.getName() + "] | ["
+ + StartupOption.CHECKPOINT.getName() + "] | ["
+ + StartupOption.FORMAT.getName() + " ["
+ + StartupOption.CLUSTERID.getName() + " cid ] ["
+ + StartupOption.FORCE.getName() + "] ["
+ + StartupOption.NONINTERACTIVE.getName() + "] ] | ["
+ + StartupOption.UPGRADE.getName() + "] | ["
+ + StartupOption.ROLLBACK.getName() + "] | ["
+ + StartupOption.FINALIZE.getName() + "] | ["
+ + StartupOption.IMPORT.getName() + "] | ["
+ + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | ["
+ + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | ["
+ + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName()
+ + " ] ]";
+
public long getProtocolVersion(String protocol,
long clientVersion) throws IOException {
if (protocol.equals(ClientProtocol.class.getName())) {
@@ -481,9 +489,9 @@ public class NameNode {
LOG.warn("ServicePlugin " + p + " could not be started", t);
}
}
- LOG.info(getRole() + " up at: " + rpcServer.getRpcAddress());
+ LOG.info(getRole() + " RPC up at: " + rpcServer.getRpcAddress());
if (rpcServer.getServiceRpcAddress() != null) {
- LOG.info(getRole() + " service server is up at: "
+ LOG.info(getRole() + " service RPC up at: "
+ rpcServer.getServiceRpcAddress());
}
}
@@ -503,18 +511,27 @@ public class NameNode {
stopHttpServer();
}
- private void startTrashEmptier(Configuration conf) throws IOException {
- long trashInterval = conf.getLong(
- CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY,
- CommonConfigurationKeys.FS_TRASH_INTERVAL_DEFAULT);
+ private void startTrashEmptier(final Configuration conf) throws IOException {
+ long trashInterval =
+ conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT);
if (trashInterval == 0) {
return;
} else if (trashInterval < 0) {
throw new IOException("Cannot start tresh emptier with negative interval."
- + " Set " + CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY + " to a"
- + " positive value.");
+ + " Set " + FS_TRASH_INTERVAL_KEY + " to a positive value.");
}
- this.emptier = new Thread(new Trash(conf).getEmptier(), "Trash Emptier");
+
+ // This may be called from the transitionToActive code path, in which
+ // case the current user is the administrator, not the NN. The trash
+ // emptier needs to run as the NN. See HDFS-3972.
+ FileSystem fs = SecurityUtil.doAsLoginUser(
+ new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return FileSystem.get(conf);
+ }
+ });
+ this.emptier = new Thread(new Trash(fs, conf).getEmptier(), "Trash Emptier");
this.emptier.setDaemon(true);
this.emptier.start();
}
@@ -611,7 +628,7 @@ public class NameNode {
*/
public void join() {
try {
- this.rpcServer.join();
+ rpcServer.join();
} catch (InterruptedException ie) {
LOG.info("Caught interrupted exception ", ie);
}
@@ -659,27 +676,31 @@ public class NameNode {
}
/**
- * Returns the address on which the NameNodes is listening to.
- * @return namenode rpc address
+ * @return NameNode RPC address
*/
public InetSocketAddress getNameNodeAddress() {
return rpcServer.getRpcAddress();
}
-
+
/**
- * Returns namenode service rpc address, if set. Otherwise returns
- * namenode rpc address.
- * @return namenode service rpc address used by datanodes
+ * @return NameNode RPC address in "host:port" string form
+ */
+ public String getNameNodeAddressHostPortString() {
+ return NetUtils.getHostPortString(rpcServer.getRpcAddress());
+ }
+
+ /**
+ * @return NameNode service RPC address if configured, the
+ * NameNode RPC address otherwise
*/
public InetSocketAddress getServiceRpcAddress() {
- return rpcServer.getServiceRpcAddress() != null ? rpcServer.getServiceRpcAddress() : rpcServer.getRpcAddress();
+ final InetSocketAddress serviceAddr = rpcServer.getServiceRpcAddress();
+ return serviceAddr == null ? rpcServer.getRpcAddress() : serviceAddr;
}
/**
- * Returns the address of the NameNodes http server,
- * which is used to access the name-node web UI.
- *
- * @return the http address.
+ * @return NameNode HTTP address, used by the Web UI, image transfer,
+ * and HTTP-based file system clients like Hftp and WebHDFS
*/
public InetSocketAddress getHttpAddress() {
return httpServer.getHttpAddress();
@@ -701,6 +722,12 @@ public class NameNode {
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
initializeGenericKeys(conf, nsId, namenodeId);
checkAllowFormat(conf);
+
+ if (UserGroupInformation.isSecurityEnabled()) {
+ InetSocketAddress socAddr = getAddress(conf);
+ SecurityUtil.login(conf, DFS_NAMENODE_KEYTAB_FILE_KEY,
+ DFS_NAMENODE_USER_NAME_KEY, socAddr.getHostName());
+ }
Collection<URI> nameDirsToFormat = FSNamesystem.getNamespaceDirs(conf);
List<URI> sharedDirs = FSNamesystem.getSharedEditsDirs(conf);
@@ -709,9 +736,6 @@ public class NameNode {
dirsToPrompt.addAll(sharedDirs);
List<URI> editDirsToFormat =
FSNamesystem.getNamespaceEditsDirs(conf);
- if (!confirmFormat(dirsToPrompt, force, isInteractive)) {
- return true; // aborted
- }
// if clusterID is not provided - see if you can find the current one
String clusterId = StartupOption.FORMAT.getClusterId();
@@ -723,62 +747,16 @@ public class NameNode {
FSImage fsImage = new FSImage(conf, nameDirsToFormat, editDirsToFormat);
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
+ fsImage.getEditLog().initJournalsForWrite();
+
+ if (!fsImage.confirmFormat(force, isInteractive)) {
+ return true; // aborted
+ }
+
fsImage.format(fsn, clusterId);
return false;
}
- /**
- * Check whether the given storage directories already exist.
- * If running in interactive mode, will prompt the user for each
- * directory to allow them to format anyway. Otherwise, returns
- * false, unless 'force' is specified.
- *
- * @param dirsToFormat the dirs to check
- * @param force format regardless of whether dirs exist
- * @param interactive prompt the user when a dir exists
- * @return true if formatting should proceed
- * @throws IOException
- */
- public static boolean confirmFormat(Collection<URI> dirsToFormat,
- boolean force, boolean interactive)
- throws IOException {
- for(Iterator<URI> it = dirsToFormat.iterator(); it.hasNext();) {
- URI dirUri = it.next();
- if (!dirUri.getScheme().equals(NNStorage.LOCAL_URI_SCHEME)) {
- System.err.println("Skipping format for directory \"" + dirUri
- + "\". Can only format local directories with scheme \""
- + NNStorage.LOCAL_URI_SCHEME + "\".");
- continue;
- }
- // To validate only file based schemes are formatted
- assert dirUri.getScheme().equals(NNStorage.LOCAL_URI_SCHEME) :
- "formatting is not supported for " + dirUri;
-
- File curDir = new File(dirUri.getPath());
- // Its alright for a dir not to exist, or to exist (properly accessible)
- // and be completely empty.
- if (!curDir.exists() ||
- (curDir.isDirectory() && FileUtil.listFiles(curDir).length == 0))
- continue;
- if (force) { // Don't confirm, always format.
- System.err.println(
- "Storage directory exists in " + curDir + ". Formatting anyway.");
- continue;
- }
- if (!interactive) { // Don't ask - always don't format
- System.err.println(
- "Running in non-interactive mode, and image appears to exist in " +
- curDir + ". Not formatting.");
- return false;
- }
- if (!confirmPrompt("Re-format filesystem in " + curDir + " ?")) {
- System.err.println("Format aborted in " + curDir);
- return false;
- }
- }
- return true;
- }
-
public static void checkAllowFormat(Configuration conf) throws IOException {
if (!conf.getBoolean(DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY,
DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT)) {
@@ -791,13 +769,13 @@ public class NameNode {
}
@VisibleForTesting
- public static boolean initializeSharedEdits(Configuration conf) {
+ public static boolean initializeSharedEdits(Configuration conf) throws IOException {
return initializeSharedEdits(conf, true);
}
@VisibleForTesting
public static boolean initializeSharedEdits(Configuration conf,
- boolean force) {
+ boolean force) throws IOException {
return initializeSharedEdits(conf, force, false);
}
@@ -811,38 +789,60 @@ public class NameNode {
* @return true if the command aborts, false otherwise
*/
private static boolean initializeSharedEdits(Configuration conf,
- boolean force, boolean interactive) {
+ boolean force, boolean interactive) throws IOException {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
initializeGenericKeys(conf, nsId, namenodeId);
+
+ if (conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY) == null) {
+ LOG.fatal("No shared edits directory configured for namespace " +
+ nsId + " namenode " + namenodeId);
+ return false;
+ }
+
+ if (UserGroupInformation.isSecurityEnabled()) {
+ InetSocketAddress socAddr = getAddress(conf);
+ SecurityUtil.login(conf, DFS_NAMENODE_KEYTAB_FILE_KEY,
+ DFS_NAMENODE_USER_NAME_KEY, socAddr.getHostName());
+ }
+
NNStorage existingStorage = null;
try {
- FSNamesystem fsns = FSNamesystem.loadFromDisk(conf,
+ Configuration confWithoutShared = new Configuration(conf);
+ confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+ FSNamesystem fsns = FSNamesystem.loadFromDisk(confWithoutShared,
FSNamesystem.getNamespaceDirs(conf),
FSNamesystem.getNamespaceEditsDirs(conf, false));
existingStorage = fsns.getFSImage().getStorage();
+ NamespaceInfo nsInfo = existingStorage.getNamespaceInfo();
- Collection<URI> sharedEditsDirs = FSNamesystem.getSharedEditsDirs(conf);
- if (!confirmFormat(sharedEditsDirs, force, interactive)) {
- return true; // aborted
- }
- NNStorage newSharedStorage = new NNStorage(conf,
+ List<URI> sharedEditsDirs = FSNamesystem.getSharedEditsDirs(conf);
+
+ FSImage sharedEditsImage = new FSImage(conf,
Lists.<URI>newArrayList(),
sharedEditsDirs);
+ sharedEditsImage.getEditLog().initJournalsForWrite();
- newSharedStorage.format(existingStorage.getNamespaceInfo());
+ if (!sharedEditsImage.confirmFormat(force, interactive)) {
+ return true; // abort
+ }
+ NNStorage newSharedStorage = sharedEditsImage.getStorage();
+ // Call Storage.format instead of FSImage.format here, since we don't
+ // actually want to save a checkpoint - just prime the dirs with
+ // the existing namespace info
+ newSharedStorage.format(nsInfo);
+ sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo);
+
// Need to make sure the edit log segments are in good shape to initialize
// the shared edits dir.
fsns.getFSImage().getEditLog().close();
fsns.getFSImage().getEditLog().initJournalsForWrite();
fsns.getFSImage().getEditLog().recoverUnclosedStreams();
-
- if (copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs,
- newSharedStorage, conf)) {
- return true; // aborted
- }
+
+ copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs, newSharedStorage,
+ conf);
} catch (IOException ioe) {
LOG.error("Could not initialize shared edits dir", ioe);
return true; // aborted
@@ -860,43 +860,59 @@ public class NameNode {
}
return false; // did not abort
}
-
- private static boolean copyEditLogSegmentsToSharedDir(FSNamesystem fsns,
+
+ private static void copyEditLogSegmentsToSharedDir(FSNamesystem fsns,
Collection<URI> sharedEditsDirs, NNStorage newSharedStorage,
- Configuration conf) throws FileNotFoundException, IOException {
+ Configuration conf) throws IOException {
+ Preconditions.checkArgument(!sharedEditsDirs.isEmpty(),
+ "No shared edits specified");
// Copy edit log segments into the new shared edits dir.
- for (JournalAndStream jas : fsns.getFSImage().getEditLog().getJournals()) {
- FileJournalManager fjm = null;
- if (!(jas.getManager() instanceof FileJournalManager)) {
- LOG.error("Cannot populate shared edits dir from non-file " +
- "journal manager: " + jas.getManager());
- return true; // aborted
- } else {
- fjm = (FileJournalManager) jas.getManager();
- }
- for (EditLogFile elf : fjm.getLogFiles(fsns.getFSImage()
- .getMostRecentCheckpointTxId())) {
- File editLogSegment = elf.getFile();
- for (URI sharedEditsUri : sharedEditsDirs) {
- StorageDirectory sharedEditsDir = newSharedStorage
- .getStorageDirectory(sharedEditsUri);
- File targetFile = new File(sharedEditsDir.getCurrentDir(),
- editLogSegment.getName());
- if (!targetFile.exists()) {
- InputStream in = null;
- OutputStream out = null;
- try {
- in = new FileInputStream(editLogSegment);
- out = new AtomicFileOutputStream(targetFile);
- IOUtils.copyBytes(in, out, conf);
- } finally {
- IOUtils.cleanup(LOG, in, out);
- }
- }
+ List<URI> sharedEditsUris = new ArrayList<URI>(sharedEditsDirs);
+ FSEditLog newSharedEditLog = new FSEditLog(conf, newSharedStorage,
+ sharedEditsUris);
+ newSharedEditLog.initJournalsForWrite();
+ newSharedEditLog.recoverUnclosedStreams();
+
+ FSEditLog sourceEditLog = fsns.getFSImage().editLog;
+
+ long fromTxId = fsns.getFSImage().getMostRecentCheckpointTxId();
+ Collection<EditLogInputStream> streams = sourceEditLog.selectInputStreams(
+ fromTxId+1, 0);
+
+ // Set the nextTxid to the CheckpointTxId+1
+ newSharedEditLog.setNextTxId(fromTxId + 1);
+
+ // Copy all edits after last CheckpointTxId to shared edits dir
+ for (EditLogInputStream stream : streams) {
+ LOG.debug("Beginning to copy stream " + stream + " to shared edits");
+ FSEditLogOp op;
+ boolean segmentOpen = false;
+ while ((op = stream.readOp()) != null) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("copying op: " + op);
}
+ if (!segmentOpen) {
+ newSharedEditLog.startLogSegment(op.txid, false);
+ segmentOpen = true;
+ }
+
+ newSharedEditLog.logEdit(op);
+
+ if (op.opCode == FSEditLogOpCodes.OP_END_LOG_SEGMENT) {
+ newSharedEditLog.logSync();
+ newSharedEditLog.endCurrentLogSegment(false);
+ LOG.debug("ending log segment because of END_LOG_SEGMENT op in " + stream);
+ segmentOpen = false;
+ }
+ }
+
+ if (segmentOpen) {
+ LOG.debug("ending log segment because of end of stream in " + stream);
+ newSharedEditLog.logSync();
+ newSharedEditLog.endCurrentLogSegment(false);
+ segmentOpen = false;
}
}
- return false; // did not abort
}
private static boolean finalize(Configuration conf,
@@ -921,22 +937,8 @@ public class NameNode {
return false;
}
- private static void printUsage() {
- System.err.println(
- "Usage: java NameNode [" +
- StartupOption.BACKUP.getName() + "] | [" +
- StartupOption.CHECKPOINT.getName() + "] | [" +
- StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() +
- " cid ] [" + StartupOption.FORCE.getName() + "] [" +
- StartupOption.NONINTERACTIVE.getName() + "] ] | [" +
- StartupOption.UPGRADE.getName() + "] | [" +
- StartupOption.ROLLBACK.getName() + "] | [" +
- StartupOption.FINALIZE.getName() + "] | [" +
- StartupOption.IMPORT.getName() + "] | [" +
- StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" +
- StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" +
- StartupOption.RECOVER.getName() + " [ " +
- StartupOption.FORCE.getName() + " ] ]");
+ private static void printUsage(PrintStream out) {
+ out.println(USAGE + "\n");
}
private static StartupOption parseArguments(String args[]) {
@@ -1003,6 +1005,16 @@ public class NameNode {
return startOpt;
} else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.INITIALIZESHAREDEDITS;
+ for (i = i + 1 ; i < argsLen; i++) {
+ if (StartupOption.NONINTERACTIVE.getName().equals(args[i])) {
+ startOpt.setInteractiveFormat(false);
+ } else if (StartupOption.FORCE.getName().equals(args[i])) {
+ startOpt.setForceFormat(true);
+ } else {
+ LOG.fatal("Invalid argument: " + args[i]);
+ return null;
+ }
+ }
return startOpt;
} else if (StartupOption.RECOVER.getName().equalsIgnoreCase(cmd)) {
if (startOpt != StartupOption.REGULAR) {
@@ -1072,9 +1084,13 @@ public class NameNode {
throws IOException {
if (conf == null)
conf = new HdfsConfiguration();
+ // Parse out some generic args into Configuration.
+ GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
+ argv = hParser.getRemainingArgs();
+ // Parse the rest, NN specific args.
StartupOption startOpt = parseArguments(argv);
if (startOpt == null) {
- printUsage();
+ printUsage(System.err);
return null;
}
setStartupOption(conf, startOpt);
@@ -1112,7 +1128,9 @@ public class NameNode {
return null; // avoid warning
}
case INITIALIZESHAREDEDITS: {
- boolean aborted = initializeSharedEdits(conf, false, true);
+ boolean aborted = initializeSharedEdits(conf,
+ startOpt.getForceFormat(),
+ startOpt.getInteractiveFormat());
terminate(aborted ? 1 : 0);
return null; // avoid warning
}
@@ -1168,10 +1186,12 @@ public class NameNode {
NAMESERVICE_SPECIFIC_KEYS);
}
+ // If the RPC address is set use it to (re-)configure the default FS
if (conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY) != null) {
URI defaultUri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://"
+ conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY));
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
+ LOG.debug("Setting " + FS_DEFAULT_NAME_KEY + " to " + defaultUri.toString());
}
}
@@ -1186,14 +1206,19 @@ public class NameNode {
/**
*/
public static void main(String argv[]) throws Exception {
+ if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
try {
StringUtils.startupShutdownMessage(NameNode.class, argv, LOG);
NameNode namenode = createNameNode(argv, null);
- if (namenode != null)
+ if (namenode != null) {
namenode.join();
+ }
} catch (Throwable e) {
LOG.fatal("Exception in namenode join", e);
- terminate(1);
+ terminate(1, e);
}
}
@@ -1281,7 +1306,7 @@ public class NameNode {
} catch (Throwable ignored) {
// This is unlikely to happen, but there's nothing we can do if it does.
}
- terminate(1, t.getMessage());
+ terminate(1, t);
}
/**
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Fri Oct 19 02:25:55 2012
@@ -18,13 +18,8 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT;
-
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.Map;
@@ -34,7 +29,6 @@ import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.AuthFilter;
@@ -55,12 +49,9 @@ public class NameNodeHttpServer {
private final Configuration conf;
private final NameNode nn;
- private final Log LOG = NameNode.LOG;
private InetSocketAddress httpAddress;
-
private InetSocketAddress bindAddress;
-
public static final String NAMENODE_ADDRESS_ATTRIBUTE_KEY = "name.node.address";
public static final String FSIMAGE_ATTRIBUTE_KEY = "name.system.image";
protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node";
@@ -74,12 +65,6 @@ public class NameNodeHttpServer {
this.bindAddress = bindAddress;
}
- private String getDefaultServerPrincipal() throws IOException {
- return SecurityUtil.getServerPrincipal(
- conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
- nn.getNameNodeAddress().getHostName());
- }
-
public void start() throws IOException {
final String infoHost = bindAddress.getHostName();
int infoPort = bindAddress.getPort();
@@ -122,13 +107,24 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
SecurityUtil.getServerPrincipal(principalInConf,
bindAddress.getHostName()));
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.error("WebHDFS and security are enabled, but configuration property '" +
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY +
+ "' is not set.");
+ }
+ String httpKeytab = conf.get(
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
+ if (httpKeytab == null) {
+ httpKeytab = conf.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY);
}
- String httpKeytab = conf
- .get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
if (httpKeytab != null && !httpKeytab.isEmpty()) {
params.put(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
httpKeytab);
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.error("WebHDFS and security are enabled, but configuration property '" +
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY +
+ "' is not set.");
}
return params;
}
@@ -152,7 +148,8 @@ public class NameNodeHttpServer {
.getPort());
}
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
- httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY, nn.getNameNodeAddress());
+ httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY,
+ NetUtils.getConnectAddress(nn.getNameNodeAddress()));
httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, nn.getFSImage());
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
setupServlets(httpServer, conf);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java Fri Oct 19 02:25:55 2012
@@ -119,6 +119,7 @@ public class NameNodeResourceChecker {
Collection<URI> localEditDirs = Collections2.filter(
FSNamesystem.getNamespaceEditsDirs(conf),
new Predicate<URI>() {
+ @Override
public boolean apply(URI input) {
if (input.getScheme().equals(NNStorage.LOCAL_URI_SCHEME)) {
return true;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourcePolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourcePolicy.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourcePolicy.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourcePolicy.java Fri Oct 19 02:25:55 2012
@@ -41,6 +41,14 @@ final class NameNodeResourcePolicy {
static boolean areResourcesAvailable(
Collection<? extends CheckableNameNodeResource> resources,
int minimumRedundantResources) {
+
+ // TODO: workaround:
+ // - during startup, if there are no edits dirs on disk, then there is
+ // a call to areResourcesAvailable() with no dirs at all, which was
+ // previously causing the NN to enter safemode
+ if (resources.isEmpty()) {
+ return true;
+ }
int requiredResourceCount = 0;
int redundantResourceCount = 0;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Fri Oct 19 02:25:55 2012
@@ -59,7 +59,6 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -83,11 +82,11 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.RefreshAuthorizationPolicyProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
@@ -106,7 +105,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
-import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -161,10 +159,11 @@ class NameNodeRpcServer implements Namen
int handlerCount =
conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY,
DFS_NAMENODE_HANDLER_COUNT_DEFAULT);
- InetSocketAddress socAddr = nn.getRpcServerAddress(conf);
- RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
- ProtobufRpcEngine.class);
- ClientNamenodeProtocolServerSideTranslatorPB
+
+ RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ ClientNamenodeProtocolServerSideTranslatorPB
clientProtocolServerTranslator =
new ClientNamenodeProtocolServerSideTranslatorPB(this);
BlockingService clientNNPbService = ClientNamenodeProtocol.
@@ -201,19 +200,24 @@ class NameNodeRpcServer implements Namen
.newReflectiveBlockingService(haServiceProtocolXlator);
WritableRpcEngine.ensureInitialized();
-
- InetSocketAddress dnSocketAddr = nn.getServiceRpcServerAddress(conf);
- if (dnSocketAddr != null) {
+
+ InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
+ if (serviceRpcAddr != null) {
int serviceHandlerCount =
conf.getInt(DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
+ serviceRpcServer = new RPC.Builder(conf)
+ .setProtocol(
+ org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(serviceRpcAddr.getHostName())
+ .setPort(serviceRpcAddr.getPort())
+ .setNumHandlers(serviceHandlerCount)
+ .setVerbose(false)
+ .setSecretManager(namesystem.getDelegationTokenSecretManager())
+ .build();
+
// Add all the RPC protocols that the namenode implements
- this.serviceRpcServer =
- RPC.getServer(org.apache.hadoop.hdfs.protocolPB.
- ClientNamenodeProtocolPB.class, clientNNPbService,
- dnSocketAddr.getHostName(), dnSocketAddr.getPort(),
- serviceHandlerCount,
- false, conf, namesystem.getDelegationTokenSecretManager());
DFSUtil.addPBProtocol(conf, HAServiceProtocolPB.class, haPbService,
serviceRpcServer);
DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService,
@@ -227,18 +231,26 @@ class NameNodeRpcServer implements Namen
DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
getUserMappingService, serviceRpcServer);
- this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress();
+ serviceRPCAddress = serviceRpcServer.getListenerAddress();
nn.setRpcServiceServerAddress(conf, serviceRPCAddress);
} else {
serviceRpcServer = null;
serviceRPCAddress = null;
}
+
+ InetSocketAddress rpcAddr = nn.getRpcServerAddress(conf);
+ clientRpcServer = new RPC.Builder(conf)
+ .setProtocol(
+ org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(rpcAddr.getHostName())
+ .setPort(rpcAddr.getPort())
+ .setNumHandlers(handlerCount)
+ .setVerbose(false)
+ .setSecretManager(namesystem.getDelegationTokenSecretManager())
+ .build();
+
// Add all the RPC protocols that the namenode implements
- this.clientRpcServer = RPC.getServer(
- org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class,
- clientNNPbService, socAddr.getHostName(),
- socAddr.getPort(), handlerCount, false, conf,
- namesystem.getDelegationTokenSecretManager());
DFSUtil.addPBProtocol(conf, HAServiceProtocolPB.class, haPbService,
clientRpcServer);
DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService,
@@ -256,41 +268,54 @@ class NameNodeRpcServer implements Namen
if (serviceAuthEnabled =
conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
- this.clientRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
- if (this.serviceRpcServer != null) {
- this.serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ clientRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ if (serviceRpcServer != null) {
+ serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
}
}
// The rpc-server port can be ephemeral... ensure we have the correct info
- this.clientRpcAddress = this.clientRpcServer.getListenerAddress();
+ clientRpcAddress = clientRpcServer.getListenerAddress();
nn.setRpcServerAddress(conf, clientRpcAddress);
- this.minimumDataNodeVersion = conf.get(
+ minimumDataNodeVersion = conf.get(
DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY,
DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT);
- }
+
+ // Set terse exception whose stack trace won't be logged
+ clientRpcServer.addTerseExceptions(SafeModeException.class);
+ }
/**
- * Actually start serving requests.
+ * Start client and service RPC servers.
*/
void start() {
- clientRpcServer.start(); //start RPC server
+ clientRpcServer.start();
if (serviceRpcServer != null) {
serviceRpcServer.start();
}
}
/**
- * Wait until the RPC server has shut down.
+ * Wait until the RPC servers have shutdown.
*/
void join() throws InterruptedException {
- this.clientRpcServer.join();
+ clientRpcServer.join();
+ if (serviceRpcServer != null) {
+ serviceRpcServer.join();
+ }
}
-
+
+ /**
+ * Stop client and service RPC servers.
+ */
void stop() {
- if(clientRpcServer != null) clientRpcServer.stop();
- if(serviceRpcServer != null) serviceRpcServer.stop();
+ if (clientRpcServer != null) {
+ clientRpcServer.stop();
+ }
+ if (serviceRpcServer != null) {
+ serviceRpcServer.stop();
+ }
}
InetSocketAddress getServiceRpcAddress() {
@@ -327,8 +352,9 @@ class NameNodeRpcServer implements Namen
namesystem.checkOperation(OperationCategory.UNCHECKED);
verifyRequest(registration);
LOG.info("Error report from " + registration + ": " + msg);
- if(errorCode == FATAL)
+ if (errorCode == FATAL) {
namesystem.releaseBackupNode(registration);
+ }
}
@Override // NamenodeProtocol
@@ -703,6 +729,13 @@ class NameNodeRpcServer implements Namen
namesystem.checkOperation(OperationCategory.UNCHECKED);
namesystem.saveNamespace();
}
+
+ @Override // ClientProtocol
+ public long rollEdits() throws AccessControlException, IOException {
+ namesystem.checkOperation(OperationCategory.JOURNAL);
+ CheckpointSignature sig = namesystem.rollEditLog();
+ return sig.getCurSegmentTxId();
+ }
@Override // ClientProtocol
public void refreshNodes() throws IOException {
@@ -739,13 +772,6 @@ class NameNodeRpcServer implements Namen
}
@Override // ClientProtocol
- public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action)
- throws IOException {
- namesystem.checkOperation(OperationCategory.READ);
- return namesystem.distributedUpgradeProgress(action);
- }
-
- @Override // ClientProtocol
public void metaSave(String filename) throws IOException {
namesystem.checkOperation(OperationCategory.UNCHECKED);
namesystem.metaSave(filename);
@@ -915,11 +941,6 @@ class NameNodeRpcServer implements Namen
return namesystem.getNamespaceInfo();
}
- @Override // DatanodeProtocol
- public UpgradeCommand processUpgradeCommand(UpgradeCommand comm) throws IOException {
- return namesystem.processDistributedUpgradeCommand(comm);
- }
-
/**
* Verifies the given registration.
*
@@ -1046,4 +1067,9 @@ class NameNodeRpcServer implements Namen
}
return clientMachine;
}
+
+ @Override
+ public DataEncryptionKey getDataEncryptionKey() throws IOException {
+ return namesystem.getBlockManager().generateDataEncryptionKey();
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Fri Oct 19 02:25:55 2012
@@ -55,6 +55,7 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
@@ -182,7 +183,7 @@ public class NamenodeFsck {
* Check files on DFS, starting from the indicated path.
*/
public void fsck() {
- final long startTime = System.currentTimeMillis();
+ final long startTime = Time.now();
try {
String msg = "FSCK started by " + UserGroupInformation.getCurrentUser()
+ " from " + remoteAddress + " for path " + path + " at " + new Date();
@@ -207,7 +208,7 @@ public class NamenodeFsck {
out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks());
out.println("FSCK ended at " + new Date() + " in "
- + (System.currentTimeMillis() - startTime + " milliseconds"));
+ + (Time.now() - startTime + " milliseconds"));
// If there were internal errors during the fsck operation, we want to
// return FAILURE_STATUS, even if those errors were not immediately
@@ -233,7 +234,7 @@ public class NamenodeFsck {
String errMsg = "Fsck on path '" + path + "' " + FAILURE_STATUS;
LOG.warn(errMsg, e);
out.println("FSCK ended at " + new Date() + " in "
- + (System.currentTimeMillis() - startTime + " milliseconds"));
+ + (Time.now() - startTime + " milliseconds"));
out.println(e.getMessage());
out.print("\n\n" + errMsg);
} finally {
@@ -559,7 +560,8 @@ public class NamenodeFsck {
block.getBlockId());
blockReader = BlockReaderFactory.newBlockReader(
conf, s, file, block, lblock
- .getBlockToken(), 0, -1);
+ .getBlockToken(), 0, -1,
+ namenode.getRpcServer().getDataEncryptionKey());
} catch (IOException ex) {
// Put chosen node into dead list, continue
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Fri Oct 19 02:25:55 2012
@@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -49,18 +48,21 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
+import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.znerd.xmlenc.XMLOutputter;
+import com.google.common.base.Preconditions;
+
class NamenodeJspHelper {
static String getSafeModeText(FSNamesystem fsn) {
if (!fsn.isInSafeMode())
@@ -118,19 +120,6 @@ class NamenodeJspHelper {
return str;
}
- static String getUpgradeStatusText(FSNamesystem fsn) {
- String statusText = "";
- try {
- UpgradeStatusReport status = fsn
- .distributedUpgradeProgress(UpgradeAction.GET_STATUS);
- statusText = (status == null ? "There are no upgrades in progress."
- : status.getStatusText(false));
- } catch (IOException e) {
- statusText = "Upgrade status unknown.";
- }
- return statusText;
- }
-
/** Return a table containing version information. */
static String getVersionTable(FSNamesystem fsn) {
return "<div class='dfstable'><table>"
@@ -139,8 +128,6 @@ class NamenodeJspHelper {
+ VersionInfo.getVersion() + ", " + VersionInfo.getRevision()
+ "</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>" + VersionInfo.getDate()
+ " by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch()
- + "</td></tr>\n <tr><td class='col1'>Upgrades:</td><td>"
- + getUpgradeStatusText(fsn)
+ "</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>" + fsn.getClusterId()
+ "</td></tr>\n <tr><td class='col1'>Block Pool ID:</td><td>" + fsn.getBlockPoolId()
+ "</td></tr>\n</table></div>";
@@ -228,6 +215,52 @@ class NamenodeJspHelper {
out.print("</table></div>\n");
}
+
+ /**
+ * Generate an HTML report containing the current status of the HDFS
+ * journals.
+ */
+ void generateJournalReport(JspWriter out, NameNode nn,
+ HttpServletRequest request) throws IOException {
+ FSEditLog log = nn.getFSImage().getEditLog();
+ Preconditions.checkArgument(log != null, "no edit log set in %s", nn);
+
+ out.println("<h3> " + nn.getRole() + " Journal Status: </h3>");
+
+ out.println("<b>Current transaction ID:</b> " +
+ nn.getFSImage().getLastAppliedOrWrittenTxId() + "<br/>");
+
+
+ boolean openForWrite = log.isOpenForWrite();
+
+ out.println("<div class=\"dfstable\">");
+ out.println("<table class=\"storage\" title=\"NameNode Journals\">\n"
+ + "<thead><tr><td><b>Journal Manager</b></td><td><b>State</b></td></tr></thead>");
+ for (JournalAndStream jas : log.getJournals()) {
+ out.print("<tr>");
+ out.print("<td>" + jas.getManager());
+ if (jas.isRequired()) {
+ out.print(" [required]");
+ }
+ out.print("</td><td>");
+
+ if (jas.isDisabled()) {
+ out.print("<span class=\"failed\">Failed</span>");
+ } else if (openForWrite) {
+ EditLogOutputStream elos = jas.getCurrentStream();
+ if (elos != null) {
+ out.println(elos.generateHtmlReport());
+ } else {
+ out.println("not currently writing");
+ }
+ } else {
+ out.println("open for read");
+ }
+ out.println("</td></tr>");
+ }
+
+ out.println("</table></div>");
+ }
void generateHealthReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
@@ -368,6 +401,7 @@ class NamenodeJspHelper {
final UserGroupInformation ugi) throws IOException, InterruptedException {
Token<DelegationTokenIdentifier> token = ugi
.doAs(new PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>() {
+ @Override
public Token<DelegationTokenIdentifier> run() throws IOException {
return nn.getDelegationToken(new Text(ugi.getUserName()));
}
@@ -409,9 +443,9 @@ class NamenodeJspHelper {
nodeToRedirect = nn.getHttpAddress().getHostName();
redirectPort = nn.getHttpAddress().getPort();
}
- String addr = NetUtils.getHostPortString(nn.getNameNodeAddress());
+ String addr = nn.getNameNodeAddressHostPortString();
String fqdn = InetAddress.getByName(nodeToRedirect).getCanonicalHostName();
- redirectLocation = "http://" + fqdn + ":" + redirectPort
+ redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":" + redirectPort
+ "/browseDirectory.jsp?namenodeInfoPort="
+ nn.getHttpAddress().getPort() + "&dir=/"
+ (tokenString == null ? "" :
@@ -460,7 +494,8 @@ class NamenodeJspHelper {
String suffix, boolean alive, int nnHttpPort, String nnaddr)
throws IOException {
// from nn_browsedfscontent.jsp:
- String url = "http://" + d.getHostName() + ":" + d.getInfoPort()
+ String url = HttpConfig.getSchemePrefix() + d.getHostName() + ":"
+ + d.getInfoPort()
+ "/browseDirectory.jsp?namenodeInfoPort=" + nnHttpPort + "&dir="
+ URLEncoder.encode("/", "UTF-8")
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnaddr);
@@ -487,7 +522,7 @@ class NamenodeJspHelper {
long decommRequestTime = d.decommissioningStatus.getStartTime();
long timestamp = d.getLastUpdate();
- long currentTime = System.currentTimeMillis();
+ long currentTime = Time.now();
long hoursSinceDecommStarted = (currentTime - decommRequestTime)/3600000;
long remainderMinutes = ((currentTime - decommRequestTime)/60000) % 60;
out.print("<td class=\"lastcontact\"> "
@@ -534,7 +569,7 @@ class NamenodeJspHelper {
String adminState = d.getAdminState().toString();
long timestamp = d.getLastUpdate();
- long currentTime = System.currentTimeMillis();
+ long currentTime = Time.now();
long bpUsed = d.getBlockPoolUsed();
String percentBpUsed = StringUtils.limitDecimalTo2(d
@@ -579,8 +614,9 @@ class NamenodeJspHelper {
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
dm.fetchDatanodes(live, dead, true);
- InetSocketAddress nnSocketAddress = (InetSocketAddress) context
- .getAttribute(NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY);
+ InetSocketAddress nnSocketAddress =
+ (InetSocketAddress)context.getAttribute(
+ NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY);
String nnaddr = nnSocketAddress.getAddress().getHostAddress() + ":"
+ nnSocketAddress.getPort();
@@ -798,7 +834,7 @@ class NamenodeJspHelper {
doc.endTag();
doc.startTag("replication");
- doc.pcdata(""+inode.getReplication());
+ doc.pcdata(""+inode.getBlockReplication());
doc.endTag();
doc.startTag("disk_space_consumed");
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java Fri Oct 19 02:25:55 2012
@@ -20,13 +20,7 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java Fri Oct 19 02:25:55 2012
@@ -68,6 +68,7 @@ public class RenewDelegationTokenServlet
try {
long result = ugi.doAs(new PrivilegedExceptionAction<Long>() {
+ @Override
public Long run() throws Exception {
return nn.getRpcServer().renewDelegationToken(token);
}