You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/01/31 04:24:27 UTC
svn commit: r1563042 - in
/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common: ./
src/main/java/ src/main/java/org/apache/hadoop/conf/
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/http/
src/main/java/org/apache/...
Author: szetszwo
Date: Fri Jan 31 03:24:24 2014
New Revision: 1563042
URL: http://svn.apache.org/r1563042
Log:
Merge r1555021 through r1563041 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
- copied unchanged from r1563041, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
Removed:
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/AdminAuthorizedServlet.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestGlobalFilter.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHtmlQuoting.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWebapps.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestPathFilter.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestServletFilter.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt Fri Jan 31 03:24:24 2014
@@ -285,9 +285,6 @@ Trunk (Unreleased)
HADOOP-9740. Fix FsShell '-text' command to be able to read Avro
files stored in HDFS and other filesystems. (Allan Yan via cutting)
- HDFS-5471. CacheAdmin -listPools fails when user lacks permissions to view
- all pools (Andrew Wang via Colin Patrick McCabe)
-
HADOOP-10044 Improve the javadoc of rpc code (sanjay Radia)
OPTIMIZATIONS
@@ -302,11 +299,50 @@ Release 2.4.0 - UNRELEASED
NEW FEATURES
+ IMPROVEMENTS
+
+ HADOOP-10139. Update and improve the Single Cluster Setup document.
+ (Akira Ajisaka via Arpit Agarwal)
+
+ HADOOP-10295. Allow distcp to automatically identify the checksum type of
+ source files and use it for the target. (jing9 and Laurent Goujon)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+Release 2.3.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
HADOOP-8545. Filesystem Implementation for OpenStack Swift
(Dmitry Mezhensky, David Dobbins, Stevel via stevel)
+ NEW FEATURES
+
IMPROVEMENTS
+ HADOOP-10046. Print a log message when SSL is enabled.
+ (David S. Wang via wang)
+
+ HADOOP-10079. log a warning message if group resolution takes too long.
+ (cmccabe)
+
+ HADOOP-9623 Update jets3t dependency to 0.9.0. (Amandeep Khurana via Colin
+ Patrick McCabe)
+
+ HADOOP-10132. RPC#stopProxy() should log the class of proxy when IllegalArgumentException
+ is encountered (Ted yu via umamahesh)
+
+ HADOOP-10248. Property name should be included in the exception where property value
+ is null (Akira AJISAKA via umamahesh)
+
+ HADOOP-10086. User document for authentication in secure cluster.
+ (Masatake Iwasaki via Arpit Agarwal)
+
+ HADOOP-10274 Lower the logging level from ERROR to WARN for UGI.doAs method
+ (Takeshi Miao via stack)
+
HADOOP-9784. Add a builder for HttpServer. (Junping Du via llu)
HADOOP 9871. Fix intermittent findbugs warnings in DefaultMetricsSystem.
@@ -427,8 +463,18 @@ Release 2.4.0 - UNRELEASED
HADOOP-9652. Allow RawLocalFs#getFileLinkStatus to fill in the link owner
and mode if requested. (Andrew Wang via Colin Patrick McCabe)
+ HADOOP-10305. Add "rpc.metrics.quantile.enable" and
+ "rpc.metrics.percentiles.intervals" to core-default.xml.
+ (Akira Ajisaka via wang)
+
+ HADOOP-10317. Rename branch-2.3 release version from 2.4.0-SNAPSHOT
+ to 2.3.0-SNAPSHOT. (wang)
+
OPTIMIZATIONS
+ HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who"
+ (vinay via cmccabe)
+
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
HADOOP-10047. Add a direct-buffer based apis for compression. (Gopal V
@@ -444,6 +490,90 @@ Release 2.4.0 - UNRELEASED
BUG FIXES
+ HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9)
+
+ HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path.
+ (Chuan Liu via cnauroth)
+
+ HADOOP-10031. FsShell -get/copyToLocal/moveFromLocal should support Windows
+ local path. (Chuan Liu via cnauroth)
+
+ HADOOP-10039. Add Hive to the list of projects using
+ AbstractDelegationTokenSecretManager. (Haohui Mai via jing9)
+
+ HADOOP-10040. hadoop.cmd in UNIX format and would not run by default on
+ Windows. (cnauroth)
+
+ HADOOP-10055. FileSystemShell.apt.vm doc has typo "numRepicas".
+ (Akira Ajisaka via cnauroth)
+
+ HADOOP-10072. TestNfsExports#testMultiMatchers fails due to non-deterministic
+ timing around cache expiry check. (cnauroth)
+
+ HADOOP-9898. Set SO_KEEPALIVE on all our sockets. (todd via wang)
+
+ HADOOP-9478. Fix race conditions during the initialization of Configuration
+ related to deprecatedKeyMap (cmccabe)
+
+ HADOOP-9660. [WINDOWS] Powershell / cmd parses -Dkey=value from command line
+ as [-Dkey, value] which breaks GenericsOptionParser.
+ (Enis Soztutar via cnauroth)
+
+ HADOOP-10078. KerberosAuthenticator always does SPNEGO. (rkanter via tucu)
+
+ HADOOP-10110. hadoop-auth has a build break due to missing dependency.
+ (Chuan Liu via arp)
+
+ HADOOP-9114. After defined the dfs.checksum.type as the NULL, write file and hflush will
+ through java.lang.ArrayIndexOutOfBoundsException (Sathish via umamahesh)
+
+ HADOOP-10130. RawLocalFS::LocalFSFileInputStream.pread does not track
+ FS::Statistics (Binglin Chang via Colin Patrick McCabe)
+
+ HDFS-5560. Trash configuration log statements prints incorrect units.
+ (Josh Elser via Andrew Wang)
+
+ HADOOP-10081. Client.setupIOStreams can leak socket resources on exception
+ or error (Tsuyoshi OZAWA via jlowe)
+
+ HADOOP-10087. UserGroupInformation.getGroupNames() fails to return primary
+ group first when JniBasedUnixGroupsMappingWithFallback is used (cmccabe)
+
+ HADOOP-10175. Har files system authority should preserve userinfo.
+ (Chuan Liu via cnauroth)
+
+ HADOOP-10090. Jobtracker metrics not updated properly after execution
+ of a mapreduce job. (ivanmi)
+
+ HADOOP-10193. hadoop-auth's PseudoAuthenticationHandler can consume getInputStream.
+ (gchanan via tucu)
+
+ HADOOP-10178. Configuration deprecation always emit "deprecated" warnings
+ when a new key is used. (Shanyu Zhao via cnauroth)
+
+ HADOOP-10234. "hadoop.cmd jar" does not propagate exit code. (cnauroth)
+
+ HADOOP-10240. Windows build instructions incorrectly state requirement of
+ protoc 2.4.1 instead of 2.5.0. (cnauroth)
+
+ HADOOP-10167. Mark hadoop-common source as UTF-8 in Maven pom files / refactoring
+ (Mikhail Antonov via cos)
+
+ HADOOP-9982. Fix dead links in hadoop site docs. (Akira Ajisaka via Arpit
+ Agarwal)
+
+ HADOOP-10212. Incorrect compile command in Native Library document.
+ (Akira Ajisaka via Arpit Agarwal)
+
+ HADOOP-9830. Fix typo at http://hadoop.apache.org/docs/current/
+ (Kousuke Saruta via Arpit Agarwal)
+
+ HADOOP-10255. Rename HttpServer to HttpServer2 to retain older
+ HttpServer in branch-2 for compatibility. (Haohui Mai via suresh)
+
+ HADOOP-10291. TestSecurityUtil#testSocketAddrWithIP fails due to test
+ order dependency. (Mit Desai via Arpit Agarwal)
+
HADOOP-9964. Fix deadlocks in TestHttpServer by synchronize
ReflectionUtils.printThreadInfo. (Junping Du via llu)
@@ -459,7 +589,6 @@ Release 2.4.0 - UNRELEASED
HADOOP-9865. FileContext#globStatus has a regression with respect to
relative path. (Chuan Lin via Colin Patrick McCabe)
-
HADOOP-9909. org.apache.hadoop.fs.Stat should permit other LANG.
(Shinichi Yamashita via Andrew Wang)
@@ -545,118 +674,8 @@ Release 2.4.0 - UNRELEASED
HADOOP-10288. Explicit reference to Log4JLogger breaks non-log4j users
(todd)
-Release 2.3.0 - UNRELEASED
-
- INCOMPATIBLE CHANGES
-
- NEW FEATURES
-
- IMPROVEMENTS
-
- HADOOP-10046. Print a log message when SSL is enabled.
- (David S. Wang via wang)
-
- HADOOP-10079. log a warning message if group resolution takes too long.
- (cmccabe)
-
- HADOOP-9623 Update jets3t dependency to 0.9.0. (Amandeep Khurana via Colin
- Patrick McCabe)
-
- HADOOP-10132. RPC#stopProxy() should log the class of proxy when IllegalArgumentException
- is encountered (Ted yu via umamahesh)
-
- HADOOP-10248. Property name should be included in the exception where property value
- is null (Akira AJISAKA via umamahesh)
-
- HADOOP-10086. User document for authentication in secure cluster.
- (Masatake Iwasaki via Arpit Agarwal)
-
- HADOOP-10274 Lower the logging level from ERROR to WARN for UGI.doAs method
- (Takeshi Miao via stack)
-
- OPTIMIZATIONS
-
- HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who"
- (vinay via cmccabe)
-
- BUG FIXES
-
- HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9)
-
- HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path.
- (Chuan Liu via cnauroth)
-
- HADOOP-10031. FsShell -get/copyToLocal/moveFromLocal should support Windows
- local path. (Chuan Liu via cnauroth)
-
- HADOOP-10039. Add Hive to the list of projects using
- AbstractDelegationTokenSecretManager. (Haohui Mai via jing9)
-
- HADOOP-10040. hadoop.cmd in UNIX format and would not run by default on
- Windows. (cnauroth)
-
- HADOOP-10055. FileSystemShell.apt.vm doc has typo "numRepicas".
- (Akira Ajisaka via cnauroth)
-
- HADOOP-10072. TestNfsExports#testMultiMatchers fails due to non-deterministic
- timing around cache expiry check. (cnauroth)
-
- HADOOP-9898. Set SO_KEEPALIVE on all our sockets. (todd via wang)
-
- HADOOP-9478. Fix race conditions during the initialization of Configuration
- related to deprecatedKeyMap (cmccabe)
-
- HADOOP-9660. [WINDOWS] Powershell / cmd parses -Dkey=value from command line
- as [-Dkey, value] which breaks GenericsOptionParser.
- (Enis Soztutar via cnauroth)
-
- HADOOP-10078. KerberosAuthenticator always does SPNEGO. (rkanter via tucu)
-
- HADOOP-10110. hadoop-auth has a build break due to missing dependency.
- (Chuan Liu via arp)
-
- HADOOP-9114. After defined the dfs.checksum.type as the NULL, write file and hflush will
- through java.lang.ArrayIndexOutOfBoundsException (Sathish via umamahesh)
-
- HADOOP-10130. RawLocalFS::LocalFSFileInputStream.pread does not track
- FS::Statistics (Binglin Chang via Colin Patrick McCabe)
-
- HDFS-5560. Trash configuration log statements prints incorrect units.
- (Josh Elser via Andrew Wang)
-
- HADOOP-10081. Client.setupIOStreams can leak socket resources on exception
- or error (Tsuyoshi OZAWA via jlowe)
-
- HADOOP-10087. UserGroupInformation.getGroupNames() fails to return primary
- group first when JniBasedUnixGroupsMappingWithFallback is used (cmccabe)
-
- HADOOP-10175. Har files system authority should preserve userinfo.
- (Chuan Liu via cnauroth)
-
- HADOOP-10090. Jobtracker metrics not updated properly after execution
- of a mapreduce job. (ivanmi)
-
- HADOOP-10193. hadoop-auth's PseudoAuthenticationHandler can consume getInputStream.
- (gchanan via tucu)
-
- HADOOP-10178. Configuration deprecation always emit "deprecated" warnings
- when a new key is used. (Shanyu Zhao via cnauroth)
-
- HADOOP-10234. "hadoop.cmd jar" does not propagate exit code. (cnauroth)
-
- HADOOP-10240. Windows build instructions incorrectly state requirement of
- protoc 2.4.1 instead of 2.5.0. (cnauroth)
-
- HADOOP-10112. har file listing doesn't work with wild card. (brandonli)
-
- HADOOP-10167. Mark hadoop-common source as UTF-8 in Maven pom files / refactoring
- (Mikhail Antonov via cos)
-
- HADOOP-9982. Fix dead links in hadoop site docs. (Akira Ajisaka via Arpit
- Agarwal)
-
- HADOOP-10212. Incorrect compile command in Native Library document.
- (Akira Ajisaka via Arpit Agarwal)
+ HADOOP-10310. SaslRpcServer should be initialized even when no secret
+ manager present. (atm)
Release 2.2.0 - 2013-10-13
Propchange: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1561944-1563041
Propchange: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1561944-1563041
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java Fri Jan 31 03:24:24 2014
@@ -27,7 +27,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
/**
* A servlet to print out the running configuration data.
@@ -47,7 +47,7 @@ public class ConfServlet extends HttpSer
*/
private Configuration getConfFromContext() {
Configuration conf = (Configuration)getServletContext().getAttribute(
- HttpServer.CONF_CONTEXT_ATTRIBUTE);
+ HttpServer2.CONF_CONTEXT_ATTRIBUTE);
assert conf != null;
return conf;
}
@@ -56,7 +56,7 @@ public class ConfServlet extends HttpSer
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
- if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
return;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Fri Jan 31 03:24:24 2014
@@ -245,6 +245,7 @@ public class CommonConfigurationKeys ext
public static final String RPC_METRICS_QUANTILE_ENABLE =
"rpc.metrics.quantile.enable";
+ public static final boolean RPC_METRICS_QUANTILE_ENABLE_DEFAULT = false;
public static final String RPC_METRICS_PERCENTILES_INTERVALS_KEY =
"rpc.metrics.percentiles.intervals";
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java Fri Jan 31 03:24:24 2014
@@ -21,21 +21,26 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.io.Writable;
/** An abstract class representing file checksums for files. */
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileChecksum implements Writable {
- /** The checksum algorithm name */
+ /** The checksum algorithm name */
public abstract String getAlgorithmName();
- /** The length of the checksum in bytes */
+ /** The length of the checksum in bytes */
public abstract int getLength();
- /** The value of the checksum in bytes */
+ /** The value of the checksum in bytes */
public abstract byte[] getBytes();
+ public ChecksumOpt getChecksumOpt() {
+ return null;
+ }
+
/** Return true if both the algorithms and the values are the same. */
@Override
public boolean equals(Object other) {
@@ -50,7 +55,7 @@ public abstract class FileChecksum imple
return this.getAlgorithmName().equals(that.getAlgorithmName())
&& Arrays.equals(this.getBytes(), that.getBytes());
}
-
+
@Override
public int hashCode() {
return getAlgorithmName().hashCode() ^ Arrays.hashCode(getBytes());
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java Fri Jan 31 03:24:24 2014
@@ -56,7 +56,7 @@ public class MD5MD5CRC32FileChecksum ext
this.crcPerBlock = crcPerBlock;
this.md5 = md5;
}
-
+
@Override
public String getAlgorithmName() {
return "MD5-of-" + crcPerBlock + "MD5-of-" + bytesPerCRC +
@@ -73,10 +73,10 @@ public class MD5MD5CRC32FileChecksum ext
throw new IOException("Unknown checksum type in " + algorithm);
}
-
+
@Override
public int getLength() {return LENGTH;}
-
+
@Override
public byte[] getBytes() {
return WritableUtils.toByteArray(this);
@@ -88,6 +88,7 @@ public class MD5MD5CRC32FileChecksum ext
return DataChecksum.Type.CRC32;
}
+ @Override
public ChecksumOpt getChecksumOpt() {
return new ChecksumOpt(getCrcType(), bytesPerCRC);
}
@@ -98,12 +99,12 @@ public class MD5MD5CRC32FileChecksum ext
crcPerBlock = in.readLong();
md5 = MD5Hash.read(in);
}
-
+
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(bytesPerCRC);
out.writeLong(crcPerBlock);
- md5.write(out);
+ md5.write(out);
}
/** Write that object to xml output. */
@@ -157,11 +158,11 @@ public class MD5MD5CRC32FileChecksum ext
}
} catch (Exception e) {
throw new SAXException("Invalid attributes: bytesPerCRC=" + bytesPerCRC
- + ", crcPerBlock=" + crcPerBlock + ", crcType=" + crcType
+ + ", crcPerBlock=" + crcPerBlock + ", crcType=" + crcType
+ ", md5=" + md5, e);
}
}
-
+
@Override
public String toString() {
return getAlgorithmName() + ":" + md5;
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/AdminAuthorizedServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/AdminAuthorizedServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/AdminAuthorizedServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/AdminAuthorizedServlet.java Fri Jan 31 03:24:24 2014
@@ -37,7 +37,7 @@ public class AdminAuthorizedServlet exte
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// Do the authorization
- if (HttpServer.hasAdministratorAccess(getServletContext(), request,
+ if (HttpServer2.hasAdministratorAccess(getServletContext(), request,
response)) {
// Authorization is done. Just call super.
super.doGet(request, response);
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Fri Jan 31 03:24:24 2014
@@ -2206,7 +2206,7 @@ public abstract class Server {
// Create the responder here
responder = new Responder();
- if (secretManager != null) {
+ if (secretManager != null || UserGroupInformation.isSecurityEnabled()) {
SaslRpcServer.init(conf);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java Fri Jan 31 03:24:24 2014
@@ -54,7 +54,8 @@ public class RpcMetrics {
int[] intervals = conf.getInts(
CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
- CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, false);
+ CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
+ CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
if (rpcQuantileEnable) {
rpcQueueTimeMillisQuantiles =
new MutableQuantiles[intervals.length];
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java Fri Jan 31 03:24:24 2014
@@ -46,7 +46,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
@@ -154,7 +154,7 @@ public class JMXJsonServlet extends Http
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) {
try {
- if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
return;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java Fri Jan 31 03:24:24 2014
@@ -28,7 +28,7 @@ import org.apache.commons.logging.*;
import org.apache.commons.logging.impl.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.util.ServletUtil;
/**
@@ -93,7 +93,7 @@ public class LogLevel {
) throws ServletException, IOException {
// Do the authorization
- if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
+ if (!HttpServer2.hasAdministratorAccess(getServletContext(), request,
response)) {
return;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java Fri Jan 31 03:24:24 2014
@@ -32,7 +32,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.metrics.spi.OutputRecord;
import org.apache.hadoop.metrics.spi.AbstractMetricsContext.MetricMap;
import org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap;
@@ -108,7 +108,7 @@ public class MetricsServlet extends Http
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
- if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
return;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java Fri Jan 31 03:24:24 2014
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.security;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
@@ -94,7 +94,7 @@ public class AuthenticationFilterInitial
}
//Resolve _HOST into bind address
- String bindAddress = conf.get(HttpServer.BIND_ADDRESS);
+ String bindAddress = conf.get(HttpServer2.BIND_ADDRESS);
String principal = filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
if (principal != null) {
try {
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Fri Jan 31 03:24:24 2014
@@ -1275,4 +1275,26 @@
Default, "dr.who=;" will consider "dr.who" as user without groups.
</description>
</property>
+
+<property>
+ <name>rpc.metrics.quantile.enable</name>
+ <value>false</value>
+ <description>
+ Setting this property to true and rpc.metrics.percentiles.intervals
+ to a comma-separated list of the granularity in seconds, the
+ 50/75/90/95/99th percentile latency for rpc queue/processing time in
+ milliseconds are added to rpc metrics.
+ </description>
+</property>
+
+<property>
+ <name>rpc.metrics.percentiles.intervals</name>
+ <value></value>
+ <description>
+ A comma-separated list of the granularity in seconds for the metrics which
+ describe the 50/75/90/95/99th percentile latency for rpc queue/processing
+ time. The metrics are outputted if rpc.metrics.quantile.enable is set to
+ true.
+ </description>
+</property>
</configuration>
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm Fri Jan 31 03:24:24 2014
@@ -20,174 +20,267 @@ Hadoop MapReduce Next Generation - Setti
%{toc|section=1|fromDepth=0}
-* Mapreduce Tarball
+* Purpose
- You should be able to obtain the MapReduce tarball from the release.
- If not, you should be able to create a tarball from the source.
+ This document describes how to set up and configure a single-node Hadoop
+ installation so that you can quickly perform simple operations using Hadoop
+ MapReduce and the Hadoop Distributed File System (HDFS).
-+---+
-$ mvn clean install -DskipTests
-$ cd hadoop-mapreduce-project
-$ mvn clean install assembly:assembly -Pnative
-+---+
- <<NOTE:>> You will need {{{http://code.google.com/p/protobuf}protoc 2.5.0}}
- installed.
+* Prerequisites
- To ignore the native builds in mapreduce you can omit the <<<-Pnative>>> argument
- for maven. The tarball should be available in <<<target/>>> directory.
+** Supported Platforms
-
-* Setting up the environment.
+ * GNU/Linux is supported as a development and production platform.
+ Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
- Assuming you have installed hadoop-common/hadoop-hdfs and exported
- <<$HADOOP_COMMON_HOME>>/<<$HADOOP_HDFS_HOME>>, untar hadoop mapreduce
- tarball and set environment variable <<$HADOOP_MAPRED_HOME>> to the
- untarred directory. Set <<$HADOOP_YARN_HOME>> the same as <<$HADOOP_MAPRED_HOME>>.
-
- <<NOTE:>> The following instructions assume you have hdfs running.
-
-* Setting up Configuration.
-
- To start the ResourceManager and NodeManager, you will have to update the configs.
- Assuming your $HADOOP_CONF_DIR is the configuration directory and has the installed
- configs for HDFS and <<<core-site.xml>>>. There are 2 config files you will have to setup
- <<<mapred-site.xml>>> and <<<yarn-site.xml>>>.
-
-** Setting up <<<mapred-site.xml>>>
-
- Add the following configs to your <<<mapred-site.xml>>>.
-
-+---+
- <property>
- <name>mapreduce.cluster.temp.dir</name>
- <value></value>
- <description>No description</description>
- <final>true</final>
- </property>
-
- <property>
- <name>mapreduce.cluster.local.dir</name>
- <value></value>
- <description>No description</description>
- <final>true</final>
- </property>
-+---+
-
-** Setting up <<<yarn-site.xml>>>
-
-Add the following configs to your <<<yarn-site.xml>>>
-
-+---+
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>host:port</value>
- <description>host is the hostname of the resource manager and
- port is the port on which the NodeManagers contact the Resource Manager.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>host:port</value>
- <description>host is the hostname of the resourcemanager and port is the port
- on which the Applications in the cluster talk to the Resource Manager.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- <description>In case you do not want to use the default scheduler</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>host:port</value>
- <description>the host is the hostname of the ResourceManager and the port is the port on
- which the clients can talk to the Resource Manager. </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value></value>
- <description>the local directories used by the nodemanager</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:port</value>
- <description>the nodemanagers bind to this port</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>10240</value>
- <description>the amount of memory on the NodeManager in GB</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- <description>directory on hdfs where the application logs are moved to </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value></value>
- <description>the directories used by Nodemanagers as log directories</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- <description>shuffle service that needs to be set for Map Reduce to run </description>
- </property>
-+---+
-
-* Setting up <<<capacity-scheduler.xml>>>
-
- Make sure you populate the root queues in <<<capacity-scheduler.xml>>>.
-
-+---+
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>unfunded,default</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.capacity</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.unfunded.capacity</name>
- <value>50</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>50</value>
- </property>
+ * Windows is also a supported platform but the followings steps
+ are for Linux only. To set up Hadoop on Windows, see
+ {{{http://wiki.apache.org/hadoop/Hadoop2OnWindows}wiki page}}.
+
+** Required Software
+
+ Required software for Linux include:
+
+ [[1]] Java\u2122 must be installed. Recommended Java versions are described
+ at {{{http://wiki.apache.org/hadoop/HadoopJavaVersions}
+ HadoopJavaVersions}}.
+
+ [[2]] ssh must be installed and sshd must be running to use the Hadoop
+ scripts that manage remote Hadoop daemons.
+
+** Installing Software
+
+ If your cluster doesn't have the requisite software you will need to install
+ it.
+
+ For example on Ubuntu Linux:
+
+----
+ $ sudo apt-get install ssh
+ $ sudo apt-get install rsync
+----
+
+* Download
+
+ To get a Hadoop distribution, download a recent stable release from one of
+ the {{{http://www.apache.org/dyn/closer.cgi/hadoop/common/}
+ Apache Download Mirrors}}.
+
+* Prepare to Start the Hadoop Cluster
+
+ Unpack the downloaded Hadoop distribution. In the distribution, edit
+ the file <<<etc/hadoop/hadoop-env.sh>>> to define some parameters as
+ follows:
+
+----
+ # set to the root of your Java installation
+ export JAVA_HOME=/usr/java/latest
+
+ # Assuming your installation directory is /usr/local/hadoop
+ export HADOOP_PREFIX=/usr/local/hadoop
+----
+
+ Try the following command:
+
+----
+ $ bin/hadoop
+----
+
+ This will display the usage documentation for the hadoop script.
+
+ Now you are ready to start your Hadoop cluster in one of the three supported
+ modes:
+
+ * {{{Standalone Operation}Local (Standalone) Mode}}
+
+ * {{{Pseudo-Distributed Operation}Pseudo-Distributed Mode}}
+
+ * {{{Fully-Distributed Operation}Fully-Distributed Mode}}
+
+* Standalone Operation
+
+ By default, Hadoop is configured to run in a non-distributed mode, as a
+ single Java process. This is useful for debugging.
+
+ The following example copies the unpacked conf directory to use as input
+ and then finds and displays every match of the given regular expression.
+ Output is written to the given output directory.
+
+----
+ $ mkdir input
+ $ cp etc/hadoop/*.xml input
+ $ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar grep input output 'dfs[a-z.]+'
+ $ cat output/*
+----
+
+* Pseudo-Distributed Operation
+
+ Hadoop can also be run on a single-node in a pseudo-distributed mode where
+ each Hadoop daemon runs in a separate Java process.
+
+** Configuration
+
+ Use the following:
+
+ etc/hadoop/core-site.xml:
+
++---+
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://localhost:9000</value>
+ </property>
+</configuration>
+---+
-* Running daemons.
-
- Assuming that the environment variables <<$HADOOP_COMMON_HOME>>, <<$HADOOP_HDFS_HOME>>, <<$HADOO_MAPRED_HOME>>,
- <<$HADOOP_YARN_HOME>>, <<$JAVA_HOME>> and <<$HADOOP_CONF_DIR>> have been set appropriately.
- Set $<<$YARN_CONF_DIR>> the same as $<<HADOOP_CONF_DIR>>
-
- Run ResourceManager and NodeManager as:
+ etc/hadoop/hdfs-site.xml:
+---+
-$ cd $HADOOP_MAPRED_HOME
-$ sbin/yarn-daemon.sh start resourcemanager
-$ sbin/yarn-daemon.sh start nodemanager
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+</configuration>
++---+
+
+** Setup passphraseless ssh
+
+ Now check that you can ssh to the localhost without a passphrase:
+
+----
+ $ ssh localhost
+----
+
+ If you cannot ssh to localhost without a passphrase, execute the
+ following commands:
+
+----
+ $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
+ $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
+----
+
+** Execution
+
+ The following instructions are to run a MapReduce job locally.
+ If you want to execute a job on YARN, see {{YARN on Single Node}}.
+
+ [[1]] Format the filesystem:
+
+----
+ $ bin/hdfs namenode -format
+----
+
+ [[2]] Start NameNode daemon and DataNode daemon:
+
+----
+ $ sbin/start-dfs.sh
+----
+
+ The hadoop daemon log output is written to the <<<${HADOOP_LOG_DIR}>>>
+ directory (defaults to <<<${HADOOP_HOME}/logs>>>).
+
+ [[3]] Browse the web interface for the NameNode; by default it is
+ available at:
+
+ * NameNode - <<<http://localhost:50070/>>>
+
+ [[4]] Make the HDFS directories required to execute MapReduce jobs:
+
+----
+ $ bin/hdfs dfs -mkdir /user
+ $ bin/hdfs dfs -mkdir /user/<username>
+----
+
+ [[5]] Copy the input files into the distributed filesystem:
+
+----
+ $ bin/hdfs dfs -put etc/hadoop input
+----
+
+ [[6]] Run some of the examples provided:
+
+----
+ $ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar grep input output 'dfs[a-z.]+'
+----
+
+ [[7]] Examine the output files:
+
+ Copy the output files from the distributed filesystem to the local
+ filesystem and examine them:
+
+----
+ $ bin/hdfs dfs -get output output
+ $ cat output/*
+----
+
+ or
+
+ View the output files on the distributed filesystem:
+
+----
+ $ bin/hdfs dfs -cat output/*
+----
+
+ [[8]] When you're done, stop the daemons with:
+
+----
+ $ sbin/stop-dfs.sh
+----
+
+** YARN on Single Node
+
+ You can run a MapReduce job on YARN in a pseudo-distributed mode by setting
+ a few parameters and running ResourceManager daemon and NodeManager daemon
+ in addition.
+
+ The following instructions assume that 1. ~ 4. steps of
+ {{{Execution}the above instructions}} are already executed.
+
+ [[1]] Configure parameters as follows:
+
+ etc/hadoop/mapred-site.xml:
+
++---+
+<configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+</configuration>
+---+
- You should be up and running. You can run randomwriter as:
+ etc/hadoop/yarn-site.xml:
+---+
-$ $HADOOP_COMMON_HOME/bin/hadoop jar hadoop-examples.jar randomwriter out
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+</configuration>
+---+
-Good luck.
+ [[2]] Start ResourceManager daemon and NodeManager daemon:
+
+----
+ $ sbin/start-yarn.sh
+----
+
+ [[3]] Browse the web interface for the ResourceManager; by default it is
+ available at:
+
+ * ResourceManager - <<<http://localhost:8088/>>>
+
+ [[4]] Run a MapReduce job.
+
+ [[5]] When you're done, stop the daemons with:
+
+----
+ $ sbin/stop-yarn.sh
+----
+
+* Fully-Distributed Operation
+
+ For information on setting up fully-distributed, non-trivial clusters
+ see {{{./ClusterSetup.html}Cluster Setup}}.
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java Fri Jan 31 03:24:24 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer.Builder;
+import org.apache.hadoop.http.HttpServer2.Builder;
import java.io.File;
import java.io.IOException;
@@ -33,7 +33,7 @@ import java.net.URL;
import java.net.MalformedURLException;
/**
- * This is a base class for functional tests of the {@link HttpServer}.
+ * This is a base class for functional tests of the {@link HttpServer2}.
* The methods are static for other classes to import statically.
*/
public class HttpServerFunctionalTest extends Assert {
@@ -54,7 +54,7 @@ public class HttpServerFunctionalTest ex
* @throws IOException if a problem occurs
* @throws AssertionError if a condition was not met
*/
- public static HttpServer createTestServer() throws IOException {
+ public static HttpServer2 createTestServer() throws IOException {
prepareTestWebapp();
return createServer(TEST);
}
@@ -68,13 +68,13 @@ public class HttpServerFunctionalTest ex
* @throws IOException if a problem occurs
* @throws AssertionError if a condition was not met
*/
- public static HttpServer createTestServer(Configuration conf)
+ public static HttpServer2 createTestServer(Configuration conf)
throws IOException {
prepareTestWebapp();
return createServer(TEST, conf);
}
- public static HttpServer createTestServer(Configuration conf, AccessControlList adminsAcl)
+ public static HttpServer2 createTestServer(Configuration conf, AccessControlList adminsAcl)
throws IOException {
prepareTestWebapp();
return createServer(TEST, conf, adminsAcl);
@@ -89,7 +89,7 @@ public class HttpServerFunctionalTest ex
* @throws IOException if a problem occurs
* @throws AssertionError if a condition was not met
*/
- public static HttpServer createTestServer(Configuration conf,
+ public static HttpServer2 createTestServer(Configuration conf,
String[] pathSpecs) throws IOException {
prepareTestWebapp();
return createServer(TEST, conf, pathSpecs);
@@ -120,10 +120,10 @@ public class HttpServerFunctionalTest ex
* @return the server
* @throws IOException if it could not be created
*/
- public static HttpServer createServer(String host, int port)
+ public static HttpServer2 createServer(String host, int port)
throws IOException {
prepareTestWebapp();
- return new HttpServer.Builder().setName(TEST)
+ return new HttpServer2.Builder().setName(TEST)
.addEndpoint(URI.create("http://" + host + ":" + port))
.setFindPort(true).build();
}
@@ -134,7 +134,7 @@ public class HttpServerFunctionalTest ex
* @return the server
* @throws IOException if it could not be created
*/
- public static HttpServer createServer(String webapp) throws IOException {
+ public static HttpServer2 createServer(String webapp) throws IOException {
return localServerBuilder(webapp).setFindPort(true).build();
}
/**
@@ -144,18 +144,18 @@ public class HttpServerFunctionalTest ex
* @return the server
* @throws IOException if it could not be created
*/
- public static HttpServer createServer(String webapp, Configuration conf)
+ public static HttpServer2 createServer(String webapp, Configuration conf)
throws IOException {
return localServerBuilder(webapp).setFindPort(true).setConf(conf).build();
}
- public static HttpServer createServer(String webapp, Configuration conf, AccessControlList adminsAcl)
+ public static HttpServer2 createServer(String webapp, Configuration conf, AccessControlList adminsAcl)
throws IOException {
return localServerBuilder(webapp).setFindPort(true).setConf(conf).setACL(adminsAcl).build();
}
private static Builder localServerBuilder(String webapp) {
- return new HttpServer.Builder().setName(webapp).addEndpoint(
+ return new HttpServer2.Builder().setName(webapp).addEndpoint(
URI.create("http://localhost:0"));
}
@@ -167,7 +167,7 @@ public class HttpServerFunctionalTest ex
* @return the server
* @throws IOException if it could not be created
*/
- public static HttpServer createServer(String webapp, Configuration conf,
+ public static HttpServer2 createServer(String webapp, Configuration conf,
String[] pathSpecs) throws IOException {
return localServerBuilder(webapp).setFindPort(true).setConf(conf).setPathSpec(pathSpecs).build();
}
@@ -180,8 +180,8 @@ public class HttpServerFunctionalTest ex
* @throws IOException on any failure
* @throws AssertionError if a condition was not met
*/
- public static HttpServer createAndStartTestServer() throws IOException {
- HttpServer server = createTestServer();
+ public static HttpServer2 createAndStartTestServer() throws IOException {
+ HttpServer2 server = createTestServer();
server.start();
return server;
}
@@ -191,7 +191,7 @@ public class HttpServerFunctionalTest ex
* @param server to stop
* @throws Exception on any failure
*/
- public static void stop(HttpServer server) throws Exception {
+ public static void stop(HttpServer2 server) throws Exception {
if (server != null) {
server.stop();
}
@@ -203,7 +203,7 @@ public class HttpServerFunctionalTest ex
* @return a URL bonded to the base of the server
* @throws MalformedURLException if the URL cannot be created.
*/
- public static URL getServerURL(HttpServer server)
+ public static URL getServerURL(HttpServer2 server)
throws MalformedURLException {
assertNotNull("No server", server);
return new URL("http://"
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestGlobalFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestGlobalFilter.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestGlobalFilter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestGlobalFilter.java Fri Jan 31 03:24:24 2014
@@ -40,7 +40,7 @@ import org.apache.hadoop.net.NetUtils;
import org.junit.Test;
public class TestGlobalFilter extends HttpServerFunctionalTest {
- static final Log LOG = LogFactory.getLog(HttpServer.class);
+ static final Log LOG = LogFactory.getLog(HttpServer2.class);
static final Set<String> RECORDS = new TreeSet<String>();
/** A very simple filter that records accessed uri's */
@@ -106,9 +106,9 @@ public class TestGlobalFilter extends Ht
Configuration conf = new Configuration();
//start a http server with CountingFilter
- conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
RecordingFilter.Initializer.class.getName());
- HttpServer http = createTestServer(conf);
+ HttpServer2 http = createTestServer(conf);
http.start();
final String fsckURL = "/fsck";
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHtmlQuoting.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHtmlQuoting.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHtmlQuoting.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHtmlQuoting.java Fri Jan 31 03:24:24 2014
@@ -68,8 +68,8 @@ public class TestHtmlQuoting {
@Test
public void testRequestQuoting() throws Exception {
HttpServletRequest mockReq = Mockito.mock(HttpServletRequest.class);
- HttpServer.QuotingInputFilter.RequestQuoter quoter =
- new HttpServer.QuotingInputFilter.RequestQuoter(mockReq);
+ HttpServer2.QuotingInputFilter.RequestQuoter quoter =
+ new HttpServer2.QuotingInputFilter.RequestQuoter(mockReq);
Mockito.doReturn("a<b").when(mockReq).getParameter("x");
assertEquals("Test simple param quoting",
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java Fri Jan 31 03:24:24 2014
@@ -51,7 +51,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.http.HttpServer.QuotingInputFilter.RequestQuoter;
+import org.apache.hadoop.http.HttpServer2.QuotingInputFilter.RequestQuoter;
import org.apache.hadoop.http.resource.JerseyResource;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Groups;
@@ -70,7 +70,7 @@ import static org.mockito.Mockito.*;
public class TestHttpServer extends HttpServerFunctionalTest {
static final Log LOG = LogFactory.getLog(TestHttpServer.class);
- private static HttpServer server;
+ private static HttpServer2 server;
private static URL baseUrl;
private static final int MAX_THREADS = 10;
@@ -150,7 +150,7 @@ public class TestHttpServer extends Http
@BeforeClass public static void setup() throws Exception {
Configuration conf = new Configuration();
- conf.setInt(HttpServer.HTTP_MAX_THREADS, 10);
+ conf.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
server = createTestServer(conf);
server.addServlet("echo", "/echo", EchoServlet.class);
server.addServlet("echomap", "/echomap", EchoMapServlet.class);
@@ -357,7 +357,7 @@ public class TestHttpServer extends Http
Configuration conf = new Configuration();
// Authorization is disabled by default
- conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
DummyFilterInitializer.class.getName());
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
MyGroupsProvider.class.getName());
@@ -366,9 +366,9 @@ public class TestHttpServer extends Http
MyGroupsProvider.mapping.put("userA", Arrays.asList("groupA"));
MyGroupsProvider.mapping.put("userB", Arrays.asList("groupB"));
- HttpServer myServer = new HttpServer.Builder().setName("test")
+ HttpServer2 myServer = new HttpServer2.Builder().setName("test")
.addEndpoint(new URI("http://localhost:0")).setFindPort(true).build();
- myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
+ myServer.setAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE, conf);
myServer.start();
String serverURL = "http://" + NetUtils.getHostPortString(myServer.getConnectorAddress(0)) + "/";
for (String servlet : new String[] { "conf", "logs", "stacks",
@@ -394,7 +394,7 @@ public class TestHttpServer extends Http
true);
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN,
true);
- conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
DummyFilterInitializer.class.getName());
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
@@ -407,10 +407,10 @@ public class TestHttpServer extends Http
MyGroupsProvider.mapping.put("userD", Arrays.asList("groupD"));
MyGroupsProvider.mapping.put("userE", Arrays.asList("groupE"));
- HttpServer myServer = new HttpServer.Builder().setName("test")
+ HttpServer2 myServer = new HttpServer2.Builder().setName("test")
.addEndpoint(new URI("http://localhost:0")).setFindPort(true).setConf(conf)
.setACL(new AccessControlList("userA,userB groupC,groupD")).build();
- myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf);
+ myServer.setAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE, conf);
myServer.start();
String serverURL = "http://"
@@ -468,39 +468,39 @@ public class TestHttpServer extends Http
Configuration conf = new Configuration();
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false);
ServletContext context = Mockito.mock(ServletContext.class);
- Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
- Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(null);
+ Mockito.when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
+ Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(null);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
Mockito.when(request.getRemoteUser()).thenReturn(null);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
//authorization OFF
- Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
+ Assert.assertTrue(HttpServer2.hasAdministratorAccess(context, request, response));
//authorization ON & user NULL
response = Mockito.mock(HttpServletResponse.class);
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
- Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response));
+ Assert.assertFalse(HttpServer2.hasAdministratorAccess(context, request, response));
Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString());
//authorization ON & user NOT NULL & ACLs NULL
response = Mockito.mock(HttpServletResponse.class);
Mockito.when(request.getRemoteUser()).thenReturn("foo");
- Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
+ Assert.assertTrue(HttpServer2.hasAdministratorAccess(context, request, response));
//authorization ON & user NOT NULL & ACLs NOT NULL & user not in ACLs
response = Mockito.mock(HttpServletResponse.class);
AccessControlList acls = Mockito.mock(AccessControlList.class);
Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false);
- Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
- Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response));
+ Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(acls);
+ Assert.assertFalse(HttpServer2.hasAdministratorAccess(context, request, response));
Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString());
//authorization ON & user NOT NULL & ACLs NOT NULL & user in in ACLs
response = Mockito.mock(HttpServletResponse.class);
Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(true);
- Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
- Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response));
+ Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(acls);
+ Assert.assertTrue(HttpServer2.hasAdministratorAccess(context, request, response));
}
@@ -508,38 +508,27 @@ public class TestHttpServer extends Http
public void testRequiresAuthorizationAccess() throws Exception {
Configuration conf = new Configuration();
ServletContext context = Mockito.mock(ServletContext.class);
- Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
+ Mockito.when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
//requires admin access to instrumentation, FALSE by default
- Assert.assertTrue(HttpServer.isInstrumentationAccessAllowed(context, request, response));
+ Assert.assertTrue(HttpServer2.isInstrumentationAccessAllowed(context, request, response));
//requires admin access to instrumentation, TRUE
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN, true);
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
AccessControlList acls = Mockito.mock(AccessControlList.class);
Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false);
- Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls);
- Assert.assertFalse(HttpServer.isInstrumentationAccessAllowed(context, request, response));
- }
-
- @Test
- @SuppressWarnings("deprecation")
- public void testOldConstructor() throws Exception {
- HttpServer server = new HttpServer("test", "0.0.0.0", 0, false);
- try {
- server.start();
- } finally {
- server.stop();
- }
+ Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(acls);
+ Assert.assertFalse(HttpServer2.isInstrumentationAccessAllowed(context, request, response));
}
@Test public void testBindAddress() throws Exception {
checkBindAddress("localhost", 0, false).stop();
// hang onto this one for a bit more testing
- HttpServer myServer = checkBindAddress("localhost", 0, false);
- HttpServer myServer2 = null;
+ HttpServer2 myServer = checkBindAddress("localhost", 0, false);
+ HttpServer2 myServer2 = null;
try {
int port = myServer.getConnectorAddress(0).getPort();
// it's already in use, true = expect a higher port
@@ -558,9 +547,9 @@ public class TestHttpServer extends Http
}
}
- private HttpServer checkBindAddress(String host, int port, boolean findPort)
+ private HttpServer2 checkBindAddress(String host, int port, boolean findPort)
throws Exception {
- HttpServer server = createServer(host, port);
+ HttpServer2 server = createServer(host, port);
try {
// not bound, ephemeral should return requested port (0 for ephemeral)
List<?> listeners = (List<?>) Whitebox.getInternalState(server,
@@ -608,7 +597,7 @@ public class TestHttpServer extends Http
public void testHttpServerBuilderWithExternalConnector() throws Exception {
Connector c = mock(Connector.class);
doReturn("localhost").when(c).getHost();
- HttpServer s = new HttpServer.Builder().setName("test").setConnector(c)
+ HttpServer2 s = new HttpServer2.Builder().setName("test").setConnector(c)
.build();
s.stop();
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java Fri Jan 31 03:24:24 2014
@@ -23,18 +23,18 @@ import org.junit.Test;
public class TestHttpServerLifecycle extends HttpServerFunctionalTest {
/**
- * Check that a server is alive by probing the {@link HttpServer#isAlive()} method
+ * Check that a server is alive by probing the {@link HttpServer2#isAlive()} method
* and the text of its toString() description
* @param server server
*/
- private void assertAlive(HttpServer server) {
+ private void assertAlive(HttpServer2 server) {
assertTrue("Server is not alive", server.isAlive());
- assertToStringContains(server, HttpServer.STATE_DESCRIPTION_ALIVE);
+ assertToStringContains(server, HttpServer2.STATE_DESCRIPTION_ALIVE);
}
- private void assertNotLive(HttpServer server) {
+ private void assertNotLive(HttpServer2 server) {
assertTrue("Server should not be live", !server.isAlive());
- assertToStringContains(server, HttpServer.STATE_DESCRIPTION_NOT_LIVE);
+ assertToStringContains(server, HttpServer2.STATE_DESCRIPTION_NOT_LIVE);
}
/**
@@ -43,12 +43,12 @@ public class TestHttpServerLifecycle ext
* @throws Throwable on failure
*/
@Test public void testCreatedServerIsNotAlive() throws Throwable {
- HttpServer server = createTestServer();
+ HttpServer2 server = createTestServer();
assertNotLive(server);
}
@Test public void testStopUnstartedServer() throws Throwable {
- HttpServer server = createTestServer();
+ HttpServer2 server = createTestServer();
stop(server);
}
@@ -59,7 +59,7 @@ public class TestHttpServerLifecycle ext
*/
@Test
public void testStartedServerIsAlive() throws Throwable {
- HttpServer server = null;
+ HttpServer2 server = null;
server = createTestServer();
assertNotLive(server);
server.start();
@@ -78,22 +78,22 @@ public class TestHttpServerLifecycle ext
requestLogAppender.setName("httprequestlog");
requestLogAppender.setFilename(System.getProperty("test.build.data", "/tmp/")
+ "jetty-name-yyyy_mm_dd.log");
- Logger.getLogger(HttpServer.class.getName() + ".test").addAppender(requestLogAppender);
- HttpServer server = null;
+ Logger.getLogger(HttpServer2.class.getName() + ".test").addAppender(requestLogAppender);
+ HttpServer2 server = null;
server = createTestServer();
assertNotLive(server);
server.start();
assertAlive(server);
stop(server);
- Logger.getLogger(HttpServer.class.getName() + ".test").removeAppender(requestLogAppender);
+ Logger.getLogger(HttpServer2.class.getName() + ".test").removeAppender(requestLogAppender);
}
/**
- * Assert that the result of {@link HttpServer#toString()} contains the specific text
+ * Assert that the result of {@link HttpServer2#toString()} contains the specific text
* @param server server to examine
* @param text text to search for
*/
- private void assertToStringContains(HttpServer server, String text) {
+ private void assertToStringContains(HttpServer2 server, String text) {
String description = server.toString();
assertTrue("Did not find \"" + text + "\" in \"" + description + "\"",
description.contains(text));
@@ -105,7 +105,7 @@ public class TestHttpServerLifecycle ext
* @throws Throwable on failure
*/
@Test public void testStoppedServerIsNotAlive() throws Throwable {
- HttpServer server = createAndStartTestServer();
+ HttpServer2 server = createAndStartTestServer();
assertAlive(server);
stop(server);
assertNotLive(server);
@@ -117,7 +117,7 @@ public class TestHttpServerLifecycle ext
* @throws Throwable on failure
*/
@Test public void testStoppingTwiceServerIsAllowed() throws Throwable {
- HttpServer server = createAndStartTestServer();
+ HttpServer2 server = createAndStartTestServer();
assertAlive(server);
stop(server);
assertNotLive(server);
@@ -133,7 +133,7 @@ public class TestHttpServerLifecycle ext
*/
@Test
public void testWepAppContextAfterServerStop() throws Throwable {
- HttpServer server = null;
+ HttpServer2 server = null;
String key = "test.attribute.key";
String value = "test.attribute.value";
server = createTestServer();
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWebapps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWebapps.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWebapps.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWebapps.java Fri Jan 31 03:24:24 2014
@@ -36,7 +36,7 @@ public class TestHttpServerWebapps exten
*/
@Test
public void testValidServerResource() throws Throwable {
- HttpServer server = null;
+ HttpServer2 server = null;
try {
server = createServer("test");
} finally {
@@ -51,7 +51,7 @@ public class TestHttpServerWebapps exten
@Test
public void testMissingServerResource() throws Throwable {
try {
- HttpServer server = createServer("NoSuchWebapp");
+ HttpServer2 server = createServer("NoSuchWebapp");
//should not have got here.
//close the server
String serverDescription = server.toString();
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestPathFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestPathFilter.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestPathFilter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestPathFilter.java Fri Jan 31 03:24:24 2014
@@ -40,7 +40,7 @@ import org.apache.hadoop.net.NetUtils;
import org.junit.Test;
public class TestPathFilter extends HttpServerFunctionalTest {
- static final Log LOG = LogFactory.getLog(HttpServer.class);
+ static final Log LOG = LogFactory.getLog(HttpServer2.class);
static final Set<String> RECORDS = new TreeSet<String>();
/** A very simple filter that records accessed uri's */
@@ -107,10 +107,10 @@ public class TestPathFilter extends Http
Configuration conf = new Configuration();
//start a http server with CountingFilter
- conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
RecordingFilter.Initializer.class.getName());
String[] pathSpecs = { "/path", "/path/*" };
- HttpServer http = createTestServer(conf, pathSpecs);
+ HttpServer2 http = createTestServer(conf, pathSpecs);
http.start();
final String baseURL = "/path";
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java Fri Jan 31 03:24:24 2014
@@ -48,7 +48,7 @@ public class TestSSLHttpServer extends H
private static final Log LOG = LogFactory.getLog(TestSSLHttpServer.class);
private static Configuration conf;
- private static HttpServer server;
+ private static HttpServer2 server;
private static URL baseUrl;
private static String keystoresDir;
private static String sslConfDir;
@@ -57,7 +57,7 @@ public class TestSSLHttpServer extends H
@BeforeClass
public static void setup() throws Exception {
conf = new Configuration();
- conf.setInt(HttpServer.HTTP_MAX_THREADS, 10);
+ conf.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
File base = new File(BASEDIR);
FileUtil.fullyDelete(base);
@@ -73,7 +73,7 @@ public class TestSSLHttpServer extends H
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, sslConf);
clientSslFactory.init();
- server = new HttpServer.Builder()
+ server = new HttpServer2.Builder()
.setName("test")
.addEndpoint(new URI("https://localhost"))
.setConf(conf)
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestServletFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestServletFilter.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestServletFilter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestServletFilter.java Fri Jan 31 03:24:24 2014
@@ -40,7 +40,7 @@ import org.apache.hadoop.test.GenericTes
import org.junit.Test;
public class TestServletFilter extends HttpServerFunctionalTest {
- static final Log LOG = LogFactory.getLog(HttpServer.class);
+ static final Log LOG = LogFactory.getLog(HttpServer2.class);
static volatile String uri = null;
/** A very simple filter which record the uri filtered. */
@@ -105,9 +105,9 @@ public class TestServletFilter extends H
Configuration conf = new Configuration();
//start a http server with CountingFilter
- conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
SimpleFilter.Initializer.class.getName());
- HttpServer http = createTestServer(conf);
+ HttpServer2 http = createTestServer(conf);
http.start();
final String fsckURL = "/fsck";
@@ -167,9 +167,9 @@ public class TestServletFilter extends H
public void testServletFilterWhenInitThrowsException() throws Exception {
Configuration conf = new Configuration();
// start a http server with ErrorFilter
- conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY,
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
ErrorFilter.Initializer.class.getName());
- HttpServer http = createTestServer(conf);
+ HttpServer2 http = createTestServer(conf);
try {
http.start();
fail("expecting exception");
@@ -186,8 +186,8 @@ public class TestServletFilter extends H
public void testContextSpecificServletFilterWhenInitThrowsException()
throws Exception {
Configuration conf = new Configuration();
- HttpServer http = createTestServer(conf);
- HttpServer.defineFilter(http.webAppContext,
+ HttpServer2 http = createTestServer(conf);
+ HttpServer2.defineFilter(http.webAppContext,
"ErrorFilter", ErrorFilter.class.getName(),
null, null);
try {
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java Fri Jan 31 03:24:24 2014
@@ -24,7 +24,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServerFunctionalTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -32,7 +32,7 @@ import org.junit.Test;
public class TestJMXJsonServlet extends HttpServerFunctionalTest {
private static final Log LOG = LogFactory.getLog(TestJMXJsonServlet.class);
- private static HttpServer server;
+ private static HttpServer2 server;
private static URL baseUrl;
@BeforeClass public static void setup() throws Exception {
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java Fri Jan 31 03:24:24 2014
@@ -20,7 +20,7 @@ package org.apache.hadoop.log;
import java.io.*;
import java.net.*;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import junit.framework.TestCase;
@@ -44,7 +44,7 @@ public class TestLogLevel extends TestCa
log.error("log.error1");
assertTrue(!Level.ERROR.equals(log.getEffectiveLevel()));
- HttpServer server = new HttpServer.Builder().setName("..")
+ HttpServer2 server = new HttpServer2.Builder().setName("..")
.addEndpoint(new URI("http://localhost:0")).setFindPort(true)
.build();
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationFilter.java Fri Jan 31 03:24:24 2014
@@ -18,7 +18,7 @@ package org.apache.hadoop.security;
import junit.framework.TestCase;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
@@ -49,7 +49,7 @@ public class TestAuthenticationFilter ex
AuthenticationFilterInitializer.SIGNATURE_SECRET_FILE,
secretFile.getAbsolutePath());
- conf.set(HttpServer.BIND_ADDRESS, "barhost");
+ conf.set(HttpServer2.BIND_ADDRESS, "barhost");
FilterContainer container = Mockito.mock(FilterContainer.class);
Mockito.doAnswer(
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Fri Jan 31 03:24:24 2014
@@ -331,7 +331,9 @@ public class TestSecurityUtil {
@Test
public void testSocketAddrWithIP() {
- verifyServiceAddr("127.0.0.1", "127.0.0.1");
+ String staticHost = "127.0.0.1";
+ NetUtils.addStaticResolution(staticHost, "localhost");
+ verifyServiceAddr(staticHost, "127.0.0.1");
}
@Test