You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by mc...@apache.org on 2005/08/01 11:04:11 UTC
svn commit: r226765 - in
/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs:
FSDirectory.java FSNamesystem.java NameNode.java
Author: mc
Date: Mon Aug 1 02:04:10 2005
New Revision: 226765
URL: http://svn.apache.org/viewcvs?rev=226765&view=rev
Log:
Modify node/name-searching inside of FSDirectory.
Use more efficient algs for name lookup&edits.
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java?rev=226765&r1=226764&r2=226765&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java Mon Aug 1 02:04:10 2005
@@ -47,7 +47,7 @@
class INode {
public String name;
public INode parent;
- public Vector children = new Vector();
+ public TreeMap children = new TreeMap();
public Block blocks[];
/**
@@ -59,61 +59,45 @@
}
/**
+ * This is the external interface
*/
INode getNode(String target) {
- if (! target.startsWith("/")) {
+ if (! target.startsWith("/") || target.length() == 0) {
return null;
- }
-
- if (parent == null) {
- if ("/".equals(target)) {
- return this;
- } else {
- // Check with children
- for (Iterator it = children.iterator(); it.hasNext(); ) {
- INode child = (INode) it.next();
- INode result = child.getNode(target);
- if (result != null) {
- return result;
- }
- }
- }
+ } else if (parent == null && "/".equals(target)) {
+ return this;
} else {
- // Strip the leading slash
- if (target.length() > 1) {
- target = target.substring(1);
+ Vector components = new Vector();
+ int start = 0;
+ int slashid = 0;
+ while (start < target.length() && (slashid = target.indexOf('/', start)) >= 0) {
+ components.add(target.substring(start, slashid));
+ start = slashid + 1;
}
-
- // Check if it's the current node
- if (name.equals(target)) {
- return this;
- }
-
- // Get the chunk up to the next slash
- String curComponent, remainder;
- int slash = target.indexOf('/');
- if (slash < 0) {
- return null;
- } else {
- curComponent = target.substring(0, slash);
- remainder = target.substring(slash);
+ if (start < target.length()) {
+ components.add(target.substring(start));
}
+ return getNode(components, 0);
+ }
+ }
- // Make sure we're on the right track
- if (! name.equals(curComponent)) {
- return null;
- }
+ /**
+ */
+ INode getNode(Vector components, int index) {
+ if (! name.equals((String) components.elementAt(index))) {
+ return null;
+ }
+ if (index == components.size()-1) {
+ return this;
+ }
- // Check with children
- for (Iterator it = children.iterator(); it.hasNext(); ) {
- INode child = (INode) it.next();
- INode result = child.getNode(remainder);
- if (result != null) {
- return result;
- }
- }
+ // Check with children
+ INode child = (INode) children.get(components.elementAt(index+1));
+ if (child == null) {
+ return null;
+ } else {
+ return child.getNode(components, index+1);
}
- return null;
}
/**
@@ -133,7 +117,7 @@
} else {
String targetName = new File(target).getName();
INode newItem = new INode(targetName, parentNode, blocks);
- parentNode.children.add(newItem);
+ parentNode.children.put(targetName, newItem);
return newItem;
}
}
@@ -146,7 +130,7 @@
if (targetNode == null) {
return null;
} else {
- targetNode.parent.children.remove(targetNode);
+ targetNode.parent.children.remove(target);
return targetNode;
}
}
@@ -155,7 +139,7 @@
*/
int numItemsInTree() {
int total = 0;
- for (Iterator it = children.iterator(); it.hasNext(); ) {
+ for (Iterator it = children.values().iterator(); it.hasNext(); ) {
INode child = (INode) it.next();
total += child.numItemsInTree();
}
@@ -188,7 +172,7 @@
*/
long computeContentsLength() {
long total = computeFileLength();
- for (Iterator it = children.iterator(); it.hasNext(); ) {
+ for (Iterator it = children.values().iterator(); it.hasNext(); ) {
INode child = (INode) it.next();
total += child.computeContentsLength();
}
@@ -202,7 +186,7 @@
v.add(this);
}
- for (Iterator it = children.iterator(); it.hasNext(); ) {
+ for (Iterator it = children.values().iterator(); it.hasNext(); ) {
INode child = (INode) it.next();
v.add(child);
}
@@ -224,7 +208,7 @@
}
}
}
- for (Iterator it = children.iterator(); it.hasNext(); ) {
+ for (Iterator it = children.values().iterator(); it.hasNext(); ) {
INode child = (INode) it.next();
child.saveImage(fullName, out);
}
@@ -510,13 +494,13 @@
INode newNode = rootDir.addNode(dst.toString(), removedNode.blocks);
if (newNode != null) {
newNode.children = removedNode.children;
- for (Iterator it = newNode.children.iterator(); it.hasNext(); ) {
+ for (Iterator it = newNode.children.values().iterator(); it.hasNext(); ) {
INode child = (INode) it.next();
child.parent = newNode;
}
return true;
} else {
- removedNode.parent.children.add(removedNode);
+ removedNode.parent.children.put(new File(dst.toString()).getName(), removedNode);
return false;
}
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java?rev=226765&r1=226764&r2=226765&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java Mon Aug 1 02:04:10 2005
@@ -393,6 +393,8 @@
}
}
return COMPLETE_SUCCESS;
+ } else {
+ System.out.println("AddFile() for " + src + " failed");
}
LOG.info("Dropped through on file add....");
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java?rev=226765&r1=226764&r2=226765&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java Mon Aug 1 02:04:10 2005
@@ -74,6 +74,8 @@
/**
*/
public LocatedBlock[] open(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
Object openResults[] = namesystem.open(new UTF8(src));
if (openResults == null) {
throw new IOException("Cannot find filename " + src);
@@ -86,11 +88,18 @@
}
return results;
}
+ } finally {
+ long end = System.currentTimeMillis();
+ opCounts++;
+ opTime += (end - start);
+ }
}
/**
*/
public LocatedBlock create(String src, String clientName, boolean overwrite) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
Object results[] = namesystem.startFile(new UTF8(src), new UTF8(clientName), overwrite);
if (results == null) {
throw new IOException("Cannot create file " + src);
@@ -99,11 +108,18 @@
DatanodeInfo targets[] = (DatanodeInfo[]) results[1];
return new LocatedBlock(b, targets);
}
+ } finally {
+ long end = System.currentTimeMillis();
+ crCounts++;
+ crTime += (end - start);
+ }
}
/**
*/
public LocatedBlock addBlock(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
Object results[] = namesystem.getAdditionalBlock(new UTF8(src));
if (results != null && results[0] == null) {
try {
@@ -122,23 +138,44 @@
DatanodeInfo targets[] = (DatanodeInfo[]) results[1];
return new LocatedBlock(b, targets);
}
+ } finally {
+ long end = System.currentTimeMillis();
+ addbCounts++;
+ addbTime += (end - start);
+ }
}
/**
*/
public void abandonBlock(Block b, String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
if (! namesystem.abandonBlock(b, new UTF8(src))) {
throw new IOException("Cannot abandon block during write to " + src);
}
+ } finally {
+ long end = System.currentTimeMillis();
+ abCounts++;
+ abTime += (end - start);
+ }
}
/**
*/
public void abandonFileInProgress(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
namesystem.abandonFileInProgress(new UTF8(src));
+ } finally {
+ long end = System.currentTimeMillis();
+ afCounts++;
+ afTime += (end - start);
+ }
}
/**
*/
public boolean complete(String src, String clientName) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
int returnCode = namesystem.completeFile(new UTF8(src), new UTF8(clientName));
if (returnCode == STILL_WAITING) {
return false;
@@ -147,10 +184,17 @@
} else {
throw new IOException("Could not complete write to file " + src + " by " + clientName);
}
+ } finally {
+ long end = System.currentTimeMillis();
+ coCounts++;
+ coTime += (end - start);
+ }
}
/**
*/
public String[] getHints(String src, long offset) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
UTF8 hosts[] = namesystem.getDatanodeHints(new UTF8(src), offset);
if (hosts == null) {
return new String[0];
@@ -161,40 +205,82 @@
}
return results;
}
+ } finally {
+ long end = System.currentTimeMillis();
+ ghCounts++;
+ ghTime += (end - start);
+ }
}
/**
*/
public boolean rename(String src, String dst) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
return namesystem.renameTo(new UTF8(src), new UTF8(dst));
+ } finally {
+ long end = System.currentTimeMillis();
+ rnCounts++;
+ rnTime += (end - start);
+ }
}
/**
*/
public boolean delete(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
return namesystem.delete(new UTF8(src));
+ } finally {
+ long end = System.currentTimeMillis();
+ deCounts++;
+ deTime += (end - start);
+ }
}
/**
*/
public boolean exists(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
return namesystem.exists(new UTF8(src));
+ } finally {
+ long end = System.currentTimeMillis();
+ exCounts++;
+ exTime += (end - start);
+ }
}
/**
*/
public boolean isDir(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
return namesystem.isDir(new UTF8(src));
+ } finally {
+ long end = System.currentTimeMillis();
+ idCounts++;
+ idTime += (end - start);
+ }
}
/**
*/
public boolean mkdirs(String src) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
return namesystem.mkdirs(new UTF8(src));
+ } finally {
+ long end = System.currentTimeMillis();
+ mdCounts++;
+ mdTime += (end - start);
+ }
}
/**
*/
public boolean obtainLock(String src, String clientName, boolean exclusive) throws IOException {
+ long start = System.currentTimeMillis();
+ try {
int returnCode = namesystem.obtainLock(new UTF8(src), new UTF8(clientName), exclusive);
if (returnCode == COMPLETE_SUCCESS) {
return true;
@@ -203,18 +289,30 @@
} else {
throw new IOException("Failure when trying to obtain lock on " + src);
}
+ } finally {
+ long end = System.currentTimeMillis();
+ olCounts++;
+ olTime += (end - start);
+ }
}
/**
*/
public boolean releaseLock(String src, String clientName) throws IOException {
- int returnCode = namesystem.releaseLock(new UTF8(src), new UTF8(clientName));
- if (returnCode == COMPLETE_SUCCESS) {
- return true;
- } else if (returnCode == STILL_WAITING) {
- return false;
- } else {
- throw new IOException("Failure when trying to release lock on " + src);
+ long start = System.currentTimeMillis();
+ try {
+ int returnCode = namesystem.releaseLock(new UTF8(src), new UTF8(clientName));
+ if (returnCode == COMPLETE_SUCCESS) {
+ return true;
+ } else if (returnCode == STILL_WAITING) {
+ return false;
+ } else {
+ throw new IOException("Failure when trying to release lock on " + src);
+ }
+ } finally {
+ long end = System.currentTimeMillis();
+ rlCounts++;
+ rlTime += (end - start);
}
}
@@ -227,12 +325,22 @@
/**
*/
public NDFSFileInfo[] getListing(String src) throws IOException {
- /**
+ System.out.println("opCounts: " + opCounts + ", avgTime: " + (opTime / (1.0 * opCounts)));
+ System.out.println("crCounts: " + crCounts + ", avgTime: " + (crTime / (1.0 * crCounts)));
+ System.out.println("addbCounts: " + addbCounts + ", avgTime: " + (addbTime / (1.0 * addbCounts)));
+ System.out.println("abCounts: " + abCounts + ", avgTime: " + (abTime / (1.0 * abCounts)));
+ System.out.println("afCounts: " + afCounts + ", avgTime: " + (afTime / (1.0 * afCounts)));
+ System.out.println("coCounts: " + coCounts + ", avgTime: " + (coTime / (1.0 * coCounts)));
+ System.out.println("ghCounts: " + ghCounts + ", avgTime: " + (ghTime / (1.0 * ghCounts)));
+ System.out.println("rnCounts: " + rnCounts + ", avgTime: " + (rnTime / (1.0 * rnCounts)));
+ System.out.println("deCounts: " + deCounts + ", avgTime: " + (deTime / (1.0 * deCounts)));
+ System.out.println("exCounts: " + exCounts + ", avgTime: " + (exTime / (1.0 * exCounts)));
+
+
System.out.println("hbCounts: " + hbCounts + ", avgTime: " + (hbTime / (1.0 * hbCounts)));
System.out.println("brCounts: " + brCounts + ", avgTime: " + (brTime / (1.0 * brCounts)));
System.out.println("brvCounts: " + brvCounts + ", avgTime: " + (brvTime / (1.0 * brvCounts)));
System.out.println("bwCounts: " + bwCounts + ", avgTime: " + (bwTime / (1.0 * bwCounts)));
- **/
return namesystem.getListing(new UTF8(src));
}
@@ -258,8 +366,8 @@
////////////////////////////////////////////////////////////////
// DatanodeProtocol
////////////////////////////////////////////////////////////////
- long hbTime = 0, brTime = 0, brvTime = 0, bwTime = 0;
- int hbCounts = 0, brCounts = 0, brvCounts = 0, bwCounts = 0;
+ long opTime = 0, crTime = 0, addbTime = 0, abTime = 0, afTime = 0, coTime = 0, ghTime = 0, rnTime = 0, deTime = 0, exTime = 0, hbTime = 0, brTime = 0, brvTime = 0, bwTime = 0, rlTime = 0, olTime = 0, mdTime = 0, idTime = 0;
+ int opCounts = 0, crCounts = 0, addbCounts = 0, abCounts = 0, afCounts = 0, coCounts = 0, ghCounts = 0, rnCounts = 0, deCounts = 0, exCounts = 0, hbCounts = 0, brCounts = 0, brvCounts = 0, bwCounts = 0, rlCounts = 0, olCounts = 0, mdCounts = 0, idCounts = 0;
/**
*/
public void sendHeartbeat(String sender, long capacity, long remaining) {