You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/07/06 20:32:07 UTC
svn commit: r1143523 [4/4] - in /hadoop/common/branches/HDFS-1073/hdfs: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/protocol/datatransfe...
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStreamFile.java Wed Jul 6 18:32:04 2011
@@ -20,17 +20,30 @@ package org.apache.hadoop.hdfs.server.na
import static org.junit.Assert.assertArrayEquals;
import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.List;
import java.util.Vector;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.mockito.Mockito;
import org.mortbay.jetty.InclusiveByteRange;
/*
@@ -186,9 +199,30 @@ class MockHttpServletResponse implements
}
+public class TestStreamFile {
+ private HdfsConfiguration CONF = new HdfsConfiguration();
+ private DFSClient clientMock = Mockito.mock(DFSClient.class);
+ private HttpServletRequest mockHttpServletRequest =
+ Mockito.mock(HttpServletRequest.class);
+ private HttpServletResponse mockHttpServletResponse =
+ Mockito.mock(HttpServletResponse.class);
+ private final ServletContext mockServletContext =
+ Mockito.mock(ServletContext.class);
-public class TestStreamFile extends TestCase {
+ StreamFile sfile = new StreamFile() {
+ private static final long serialVersionUID = -5513776238875189473L;
+ public ServletContext getServletContext() {
+ return mockServletContext;
+ }
+
+ @Override
+ protected DFSClient getDFSClient(HttpServletRequest request)
+ throws IOException, InterruptedException {
+ return clientMock;
+ }
+ };
+
// return an array matching the output of mockfsinputstream
private static byte[] getOutputArray(int start, int count) {
byte[] a = new byte[count];
@@ -200,6 +234,7 @@ public class TestStreamFile extends Test
return a;
}
+ @Test
public void testWriteTo() throws IOException, InterruptedException {
FSInputStream fsin = new MockFSInputStream();
@@ -219,7 +254,7 @@ public class TestStreamFile extends Test
assertTrue("Pairs array must be even", pairs.length % 2 == 0);
for (int i = 0; i < pairs.length; i+=2) {
- StreamFile.writeTo(fsin, os, pairs[i], pairs[i+1]);
+ StreamFile.copyFromOffset(fsin, os, pairs[i], pairs[i+1]);
assertArrayEquals("Reading " + pairs[i+1]
+ " bytes from offset " + pairs[i],
getOutputArray(pairs[i], pairs[i+1]),
@@ -228,20 +263,22 @@ public class TestStreamFile extends Test
}
}
-
- private List<?> strToRanges(String s, int contentLength) {
+
+ @SuppressWarnings("unchecked")
+ private List<InclusiveByteRange> strToRanges(String s, int contentLength) {
List<String> l = Arrays.asList(new String[]{"bytes="+s});
Enumeration<?> e = (new Vector<String>(l)).elements();
return InclusiveByteRange.satisfiableRanges(e, contentLength);
}
+ @Test
public void testSendPartialData() throws IOException, InterruptedException {
FSInputStream in = new MockFSInputStream();
ByteArrayOutputStream os = new ByteArrayOutputStream();
// test if multiple ranges, then 416
{
- List<?> ranges = strToRanges("0-,10-300", 500);
+ List<InclusiveByteRange> ranges = strToRanges("0-,10-300", 500);
MockHttpServletResponse response = new MockHttpServletResponse();
StreamFile.sendPartialData(in, os, response, 500, ranges);
assertEquals("Multiple ranges should result in a 416 error",
@@ -259,7 +296,7 @@ public class TestStreamFile extends Test
// test if invalid single range (out of bounds), then 416
{
- List<?> ranges = strToRanges("600-800", 500);
+ List<InclusiveByteRange> ranges = strToRanges("600-800", 500);
MockHttpServletResponse response = new MockHttpServletResponse();
StreamFile.sendPartialData(in, os, response, 500, ranges);
assertEquals("Single (but invalid) range should result in a 416",
@@ -269,7 +306,7 @@ public class TestStreamFile extends Test
// test if one (valid) range, then 206
{
- List<?> ranges = strToRanges("100-300", 500);
+ List<InclusiveByteRange> ranges = strToRanges("100-300", 500);
MockHttpServletResponse response = new MockHttpServletResponse();
StreamFile.sendPartialData(in, os, response, 500, ranges);
assertEquals("Single (valid) range should result in a 206",
@@ -280,4 +317,108 @@ public class TestStreamFile extends Test
}
}
+
+
+ // Test for positive scenario
+ @Test
+ public void testDoGetShouldWriteTheFileContentIntoServletOutputStream()
+ throws Exception {
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(1)
+ .build();
+ try {
+ Path testFile = createFile();
+ setUpForDoGetTest(cluster, testFile);
+ ServletOutputStreamExtn outStream = new ServletOutputStreamExtn();
+ Mockito.doReturn(outStream).when(mockHttpServletResponse)
+ .getOutputStream();
+ StreamFile sfile = new StreamFile() {
+
+ private static final long serialVersionUID = 7715590481809562722L;
+
+ public ServletContext getServletContext() {
+ return mockServletContext;
+ }
+ };
+ sfile.doGet(mockHttpServletRequest, mockHttpServletResponse);
+ assertEquals("Not writing the file data into ServletOutputStream",
+ outStream.getResult(), "test");
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ // Test for cleaning the streams in exception cases also
+ @Test
+ public void testDoGetShouldCloseTheDFSInputStreamIfResponseGetOutPutStreamThrowsAnyException()
+ throws Exception {
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(1)
+ .build();
+ try {
+ Path testFile = createFile();
+
+ setUpForDoGetTest(cluster, testFile);
+
+ Mockito.doThrow(new IOException()).when(mockHttpServletResponse)
+ .getOutputStream();
+ DFSInputStream fsMock = Mockito.mock(DFSInputStream.class);
+
+ Mockito.doReturn(fsMock).when(clientMock).open(testFile.toString());
+
+ Mockito.doReturn(Long.valueOf(4)).when(fsMock).getFileLength();
+
+ try {
+ sfile.doGet(mockHttpServletRequest, mockHttpServletResponse);
+ fail("Not throwing the IOException");
+ } catch (IOException e) {
+ Mockito.verify(clientMock, Mockito.atLeastOnce()).close();
+ }
+
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ private void setUpForDoGetTest(MiniDFSCluster cluster, Path testFile)
+ throws IOException {
+
+ Mockito.doReturn(CONF).when(mockServletContext).getAttribute(
+ JspHelper.CURRENT_CONF);
+ Mockito.doReturn(NameNode.getHostPortString(NameNode.getAddress(CONF)))
+ .when(mockHttpServletRequest).getParameter("nnaddr");
+ Mockito.doReturn(testFile.toString()).when(mockHttpServletRequest)
+ .getPathInfo();
+ }
+
+ static Path writeFile(FileSystem fs, Path f) throws IOException {
+ DataOutputStream out = fs.create(f);
+ try {
+ out.writeBytes("test");
+ } finally {
+ out.close();
+ }
+ assertTrue(fs.exists(f));
+ return f;
+ }
+
+ private Path createFile() throws IOException {
+ FileSystem fs = FileSystem.get(CONF);
+ Path testFile = new Path("/test/mkdirs/doGet");
+ writeFile(fs, testFile);
+ return testFile;
+ }
+
+ public static class ServletOutputStreamExtn extends ServletOutputStream {
+ private StringBuffer buffer = new StringBuffer(3);
+
+ public String getResult() {
+ return buffer.toString();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ buffer.append((char) b);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java Wed Jul 6 18:32:04 2011
@@ -17,34 +17,35 @@
*/
package org.apache.hadoop.hdfs.server.namenode.metrics;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+
import java.io.DataInputStream;
import java.io.IOException;
import java.util.Random;
import junit.framework.TestCase;
+import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.server.namenode.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.MetricsAsserts;
import org.apache.log4j.Level;
-import org.apache.commons.logging.LogFactory;
-
-import static org.apache.hadoop.test.MetricsAsserts.*;
-
/**
* Test for metrics published by the Namenode
*/
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/net/TestNetworkTopology.java Wed Jul 6 18:32:04 2011
@@ -24,8 +24,8 @@ import java.util.Map;
import junit.framework.TestCase;
-import org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
public class TestNetworkTopology extends TestCase {
private final static NetworkTopology cluster = new NetworkTopology();
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java Wed Jul 6 18:32:04 2011
@@ -19,6 +19,16 @@
package org.apache.hadoop.hdfs.server.namenode;
import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -34,20 +44,16 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.After;
-
-import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.*;
-
-import java.io.File;
-import java.io.IOException;
public class TestNNLeaseRecovery {
private static final Log LOG = LogFactory.getLog(TestNNLeaseRecovery.class);
Propchange: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 18:32:04 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hdfs/src/webapps/datanode:1134994-1138149
+/hadoop/common/trunk/hdfs/src/webapps/datanode:1134994-1143516
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:987665-1095512
Propchange: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 18:32:04 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hdfs/src/webapps/hdfs:1134994-1138149
+/hadoop/common/trunk/hdfs/src/webapps/hdfs:1134994-1143516
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:987665-1095512
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp?rev=1143523&r1=1143522&r2=1143523&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/webapps/hdfs/block_info_xml.jsp Wed Jul 6 18:32:04 2011
@@ -59,19 +59,9 @@
%>
<%@ page
contentType="application/xml"
- import="java.io.IOException"
- import="java.util.Iterator"
- import="org.apache.hadoop.conf.Configuration"
- import="org.apache.hadoop.hdfs.protocol.Block"
- import="org.apache.hadoop.hdfs.server.namenode.INode"
- import="org.apache.hadoop.hdfs.server.namenode.BlocksMap"
- import="org.apache.hadoop.hdfs.server.namenode.BlockInfo"
- import="org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor"
import="org.apache.hadoop.hdfs.server.namenode.NamenodeJspHelper.XMLBlockInfo"
import="org.apache.hadoop.hdfs.server.common.JspHelper"
- import="org.apache.hadoop.util.ServletUtil"
import="org.znerd.xmlenc.*"
-
%>
<%!
//for java.io.Serializable
Propchange: hadoop/common/branches/HDFS-1073/hdfs/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 18:32:04 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hdfs/src/webapps/secondary:1134994-1138149
+/hadoop/common/trunk/hdfs/src/webapps/secondary:1134994-1143516
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:987665-1095512