You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2007/10/19 06:56:53 UTC
svn commit: r586262 - in /lucene/hadoop/trunk: ./
src/java/org/apache/hadoop/dfs/ src/webapps/dfs/ src/webapps/static/
Author: dhruba
Date: Thu Oct 18 21:56:51 2007
New Revision: 586262
URL: http://svn.apache.org/viewvc?rev=586262&view=rev
Log:
HADOOP-1855. fsck verifies block placement policies and reports
violations. (Konstantin Shvachko via dhruba)
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSAdmin.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/JspHelper.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeStatusReport.java
lucene/hadoop/trunk/src/webapps/dfs/dfshealth.jsp
lucene/hadoop/trunk/src/webapps/static/hadoop.css
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Oct 18 21:56:51 2007
@@ -11,6 +11,9 @@
HADOOP-2058. Facilitate creating new datanodes dynamically in
MiniDFSCluster. (Hairong Kuang via dhruba)
+ HADOOP-1855. fsck verifies block placement policies and reports
+ violations. (Konstantin Shvachko via dhruba)
+
Branch 0.15 (unreleased changes)
INCOMPATIBLE CHANGES
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java Thu Oct 18 21:56:51 2007
@@ -1220,7 +1220,7 @@
public BlockCrcUpgradeStatusReport(int version, short status,
String extraText) {
- super(version, status);
+ super(version, status, false);
this.extraText = extraText;
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSAdmin.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSAdmin.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSAdmin.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSAdmin.java Thu Oct 18 21:56:51 2007
@@ -292,7 +292,7 @@
DistributedFileSystem dfs = (DistributedFileSystem) fs;
UpgradeStatusReport status = dfs.distributedUpgradeProgress(action);
String statusText = (status == null ?
- "There are no distributed upgrades in progress." :
+ "There are no upgrades in progress." :
status.getStatusText(action == UpgradeAction.DETAILED_STATUS));
System.out.println(statusText);
return 0;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java Thu Oct 18 21:56:51 2007
@@ -135,7 +135,7 @@
public enum SafeModeAction{ SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET; }
// Startup options
- public enum StartupOption{ FORMAT, REGULAR, UPGRADE, ROLLBACK; }
+ public enum StartupOption{ FORMAT, REGULAR, UPGRADE, ROLLBACK, FINALIZE; }
// type of the datanode report
public static enum DatanodeReportType {ALL, LIVE, DEAD }
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Thu Oct 18 21:56:51 2007
@@ -32,7 +32,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
-import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.lang.Math;
@@ -340,12 +339,16 @@
private void doFinalize(StorageDirectory sd) throws IOException {
File prevDir = sd.getPreviousDir();
- if (!prevDir.exists())
- return; // already discarded
+ if (!prevDir.exists()) { // already discarded
+ LOG.info("Directory " + prevDir + " does not exist.");
+ LOG.info("Finalize upgrade for " + sd.root + " is not required.");
+ return;
+ }
LOG.info("Finalizing upgrade for storage directory "
- + sd.root
- + ".\n cur LV = " + this.getLayoutVersion()
- + "; cur CTime = " + this.getCTime());
+ + sd.root + "."
+ + (getLayoutVersion()==0 ? "" :
+ "\n cur LV = " + this.getLayoutVersion()
+ + "; cur CTime = " + this.getCTime()));
assert sd.getCurrentDir().exists() : "Current directory must exist.";
final File tmpDir = sd.getFinalizedTmp();
// rename previous to tmp and remove
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/JspHelper.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/JspHelper.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/JspHelper.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/JspHelper.java Thu Oct 18 21:56:51 2007
@@ -32,6 +32,7 @@
import javax.servlet.jsp.JspWriter;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.UpgradeAction;
import org.apache.hadoop.fs.Path;
public class JspHelper {
@@ -169,7 +170,21 @@
return "";
return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
}
-
+
+ public String getUpgradeStatusText() {
+ String statusText = "";
+ try {
+ UpgradeStatusReport status =
+ fsn.distributedUpgradeProgress(UpgradeAction.GET_STATUS);
+ statusText = (status == null ?
+ "There are no upgrades in progress." :
+ status.getStatusText(false));
+ } catch(IOException e) {
+ statusText = "Upgrade status unknown.";
+ }
+ return statusText;
+ }
+
public void sortNodeList(ArrayList<DatanodeDescriptor> nodes,
String field, String order) {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/NameNode.java Thu Oct 18 21:56:51 2007
@@ -742,9 +742,30 @@
return false;
}
+ private static boolean finalize(Configuration conf,
+ boolean isConfirmationNeeded
+ ) throws IOException {
+ Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
+ FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat), conf);
+ System.err.print(
+ "\"finalize\" will remove the previous state of the files system.\n"
+ + "Recent upgrade will become permanent.\n"
+ + "Rollback option will not be available anymore.\n");
+ if (isConfirmationNeeded) {
+ System.err.print("Finalize filesystem state ? (Y or N) ");
+ if (!(System.in.read() == 'Y')) {
+ System.err.println("Finalize aborted.");
+ return true;
+ }
+ while(System.in.read() != '\n'); // discard the enter-key
+ }
+ nsys.dir.fsImage.finalizeUpgrade();
+ return false;
+ }
+
private static void printUsage() {
System.err.println(
- "Usage: java NameNode [-format] | [-upgrade] | [-rollback]");
+ "Usage: java NameNode [-format] | [-upgrade] | [-rollback] | [-finalize]");
}
private static StartupOption parseArguments(String args[],
@@ -761,6 +782,8 @@
startOpt = StartupOption.UPGRADE;
} else if ("-rollback".equalsIgnoreCase(cmd)) {
startOpt = StartupOption.ROLLBACK;
+ } else if ("-finalize".equalsIgnoreCase(cmd)) {
+ startOpt = StartupOption.FINALIZE;
} else
return null;
}
@@ -786,12 +809,17 @@
printUsage();
return null;
}
-
- if (startOpt == StartupOption.FORMAT) {
- boolean aborted = format(conf, true);
- System.exit(aborted ? 1 : 0);
+
+ switch (startOpt) {
+ case FORMAT:
+ boolean aborted = format(conf, true);
+ System.exit(aborted ? 1 : 0);
+ case FINALIZE:
+ aborted = finalize(conf, true);
+ System.exit(aborted ? 1 : 0);
+ default:
}
-
+
NameNode namenode = new NameNode(conf);
return namenode;
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java Thu Oct 18 21:56:51 2007
@@ -104,10 +104,17 @@
FSNamesystem.getFSNamesystem().leaveSafeMode(false);
}
- UpgradeStatusReport distributedUpgradeProgress(FSConstants.UpgradeAction action
+ UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action
) throws IOException {
- if(currentUpgrades == null)
- return null; // no upgrades are in progress
+ boolean isFinalized = false;
+ if(currentUpgrades == null) { // no upgrades are in progress
+ FSImage fsimage = FSNamesystem.getFSNamesystem().getFSImage();
+ isFinalized = fsimage.isUpgradeFinalized();
+ if(isFinalized) // upgrade is finalized
+ return null; // nothing to report
+ return new UpgradeStatusReport(fsimage.getLayoutVersion(),
+ (short)101, isFinalized);
+ }
UpgradeObjectNamenode curUO = (UpgradeObjectNamenode)currentUpgrades.first();
boolean details = false;
switch(action) {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java Thu Oct 18 21:56:51 2007
@@ -40,7 +40,7 @@
public UpgradeStatusReport getUpgradeStatusReport(boolean details)
throws IOException {
- return new UpgradeStatusReport(getVersion(), getUpgradeStatus());
+ return new UpgradeStatusReport(getVersion(), getUpgradeStatus(), false);
}
public int compareTo(Upgradeable o) {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeStatusReport.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeStatusReport.java?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeStatusReport.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/UpgradeStatusReport.java Thu Oct 18 21:56:51 2007
@@ -34,15 +34,18 @@
public class UpgradeStatusReport implements Writable {
protected int version;
protected short upgradeStatus;
+ protected boolean finalized;
public UpgradeStatusReport() {
this.version = 0;
this.upgradeStatus = 0;
+ this.finalized = false;
}
- public UpgradeStatusReport( int version, short status) {
+ public UpgradeStatusReport(int version, short status, boolean isFinalized) {
this.version = version;
this.upgradeStatus = status;
+ this.finalized = isFinalized;
}
/**
@@ -63,6 +66,14 @@
}
/**
+ * Is current upgrade finalized.
+ * @return true if finalized or false otherwise.
+ */
+ public boolean isFinalized() {
+ return this.finalized;
+ }
+
+ /**
* Get upgradeStatus data as a text for reporting.
* Should be overloaded for a particular upgrade specific upgradeStatus data.
*
@@ -71,8 +82,12 @@
* @return text
*/
public String getStatusText(boolean details) {
- return "Distributed upgrade for version " + getVersion()
- + " is in progress. Status = " + getUpgradeStatus() + "%";
+ return "Upgrade for version " + getVersion()
+ + (upgradeStatus<100 ?
+ " is in progress. Status = " + upgradeStatus + "%" :
+ " has been completed."
+ + "\nUpgrade is " + (finalized ? "" : "not ")
+ + "finalized.");
}
/**
Modified: lucene/hadoop/trunk/src/webapps/dfs/dfshealth.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/dfs/dfshealth.jsp?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/dfs/dfshealth.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/dfs/dfshealth.jsp Thu Oct 18 21:56:51 2007
@@ -230,10 +230,9 @@
<div id="dfstable"> <table>
<tr> <td id="col1"> Started: <td> <%= fsn.getStartTime()%>
-<tr> <td id="col1"> Version: <td> <%= VersionInfo.getVersion()%>,
- r<%= VersionInfo.getRevision()%>
-<tr> <td id="col1"> Compiled: <td> <%= VersionInfo.getDate()%> by
- <%= VersionInfo.getUser()%>
+<tr> <td id="col1"> Version: <td> <%= VersionInfo.getVersion()%>, r<%= VersionInfo.getRevision()%>
+<tr> <td id="col1"> Compiled: <td> <%= VersionInfo.getDate()%> by <%= VersionInfo.getUser()%>
+<tr> <td id="col1"> Upgrades: <td> <%= jspHelper.getUpgradeStatusText()%>
</table></div><br>
<b><a href="/nn_browsedfscontent.jsp">Browse the filesystem</a></b>
Modified: lucene/hadoop/trunk/src/webapps/static/hadoop.css
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/static/hadoop.css?rev=586262&r1=586261&r2=586262&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/static/hadoop.css (original)
+++ lucene/hadoop/trunk/src/webapps/static/hadoop.css Thu Oct 18 21:56:51 2007
@@ -7,6 +7,10 @@
font-weight : bolder;
}
+div#dfstable td#col1 {
+ vertical-align : top;
+}
+
div#dfstable td#col3 {
text-align : right;
}
@@ -52,7 +56,7 @@
}
div#dfstable table {
- width : 40%;
+ white-space : pre;
}
div#dfsnodetable td, div#dfsnodetable th, div#dfstable td {