You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2013/03/18 12:45:10 UTC
svn commit: r1457716 - in
/hadoop/common/branches/HDFS-2802/hadoop-common-project:
hadoop-annotations/ hadoop-common/ hadoop-common/src/main/docs/
hadoop-common/src/main/java/
hadoop-common/src/main/java/org/apache/hadoop/fs/shell/
hadoop-common/src/ma...
Author: szetszwo
Date: Mon Mar 18 11:45:07 2013
New Revision: 1457716
URL: http://svn.apache.org/r1457716
Log:
Merge r1455389 through r1457712 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java
- copied unchanged from r1457712, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
- copied unchanged from r1457712, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-annotations/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-annotations/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-annotations/pom.xml?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-annotations/pom.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-annotations/pom.xml Mon Mar 18 11:45:07 2013
@@ -38,4 +38,24 @@
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>os.linux</id>
+ <activation>
+ <os>
+ <family>!Mac</family>
+ </os>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ <version>1.6</version>
+ <scope>system</scope>
+ <systemPath>${java.home}/../lib/tools.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt Mon Mar 18 11:45:07 2013
@@ -340,6 +340,19 @@ Trunk (Unreleased)
HADOOP-9264. Port change to use Java untar API on Windows from
branch-1-win to trunk. (Chris Nauroth via suresh)
+ HADOOP-9393. TestRPC fails with JDK7. (Andrew Wang via atm)
+
+ HADOOP-9394. Port findHangingTest.sh from HBase to Hadoop. (Andrew Wang
+ via atm)
+
+ HADOOP-9099. NetUtils.normalizeHostName fails on domains where
+ UnknownHost resolves to an IP address. (Ivan Mitic via suresh)
+
+ HADOOP-9397. Incremental dist tar build fails (Chris Nauroth via jlowe)
+
+ HADOOP-9405. TestGridmixSummary#testExecutionSummarizer is broken. (Andrew
+ Wang via atm)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -468,6 +481,11 @@ Trunk (Unreleased)
HADOOP-9364. PathData#expandAsGlob does not return correct results for
absolute paths on Windows. (Ivan Mitic via suresh)
+ HADOOP-8973. DiskChecker cannot reliably detect an inaccessible disk on
+ Windows with NTFS ACLs. (Chris Nauroth via suresh)
+
+ HADOOP-9388. TestFsShellCopy fails on Windows. (Ivan Mitic via suresh)
+
Release 2.0.5-beta - UNRELEASED
INCOMPATIBLE CHANGES
@@ -493,6 +511,9 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9343. Allow additional exceptions through the RPC layer. (sseth)
+ HADOOP-9318. When exiting on a signal, print the signal name first. (Colin
+ Patrick McCabe via atm)
+
OPTIMIZATIONS
BUG FIXES
@@ -545,6 +566,11 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9379. capture the ulimit info after printing the log to the
console. (Arpit Gupta via suresh)
+ HADOOP-9399. protoc maven plugin doesn't work on mvn 3.0.2 (todd)
+
+ HADOOP-9407. commons-daemon 1.0.3 dependency has bad group id causing
+ build issues. (Sangjin Lee via suresh)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
@@ -557,6 +583,14 @@ Release 2.0.4-alpha - UNRELEASED
BUG FIXES
+ HADOOP-9406. hadoop-client leaks dependency on JDK tools jar. (tucu)
+
+ HADOOP-9301. hadoop client servlet/jsp/jetty/tomcat JARs creating
+ conflicts in Oozie & HttpFS. (tucu)
+
+ HADOOP-9408. misleading description for net.topology.table.file.name
+ property in core-default.xml. (rajeshbabu via suresh)
+
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES
@@ -2798,6 +2832,9 @@ Release 0.23.0 - 2011-11-01
HADOOP-7797. Fix top-level pom.xml to refer to correct staging maven
repository. (omalley via acmurthy)
+ HADOOP-7101. UserGroupInformation.getCurrentUser() fails when called from
+ non-Hadoop JAAS context. (todd)
+
Release 0.22.1 - Unreleased
INCOMPATIBLE CHANGES
@@ -3255,9 +3292,6 @@ Release 0.22.0 - 2011-11-29
HADOOP-7093. Servlets should default to text/plain (todd)
- HADOOP-7101. UserGroupInformation.getCurrentUser() fails when called from
- non-Hadoop JAAS context. (todd)
-
HADOOP-7089. Fix link resolution logic in hadoop-config.sh. (eli)
HADOOP-7046. Fix Findbugs warning in Configuration. (Po Cheung via shv)
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1455389-1457712
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/pom.xml?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/pom.xml Mon Mar 18 11:45:07 2013
@@ -224,6 +224,10 @@
<artifactId>jline</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
<!-- otherwise seems to drag in junit 3.8.1 via jline -->
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1455389-1457712
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1455389-1457712
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Mon Mar 18 11:45:07 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.shell;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
@@ -224,7 +223,13 @@ abstract class CommandWithDestination ex
*/
protected void copyFileToTarget(PathData src, PathData target) throws IOException {
src.fs.setVerifyChecksum(verifyChecksum);
- copyStreamToTarget(src.fs.open(src.path), target);
+ InputStream in = null;
+ try {
+ in = src.fs.open(src.path);
+ copyStreamToTarget(in, target);
+ } finally {
+ IOUtils.closeStream(in);
+ }
}
/**
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Mon Mar 18 11:45:07 2013
@@ -35,6 +35,7 @@ import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang.SystemUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
@@ -613,6 +614,13 @@ public class StringUtils {
)
);
+ if (SystemUtils.IS_OS_UNIX) {
+ try {
+ SignalLogger.INSTANCE.register(LOG);
+ } catch (Throwable t) {
+ LOG.warn("failed to register any UNIX signal loggers: ", t);
+ }
+ }
ShutdownHookManager.get().addShutdownHook(
new Runnable() {
@Override
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Mon Mar 18 11:45:07 2013
@@ -699,7 +699,7 @@
<name>net.topology.table.file.name</name>
<value></value>
<description> The file name for a topology file, which is used when the
- net.topology.script.file.name property is set to
+ net.topology.node.switch.mapping.impl property is set to
org.apache.hadoop.net.TableMapping. The file format is a two column text
file, with columns separated by whitespace. The first column is a DNS or
IP address and the second column specifies the rack where the address maps.
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1455389-1457712
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java Mon Mar 18 11:45:07 2013
@@ -834,23 +834,27 @@ public class TestRPC {
TestProtocol.class, new TestImpl(), ADDRESS, 0, 5, true, conf, null
);
server.start();
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
- final TestProtocol proxy = (TestProtocol) RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID, addr, conf);
- // Connect to the server
- proxy.ping();
- // Interrupt self, try another call
- Thread.currentThread().interrupt();
try {
+ InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
+ final TestProtocol proxy = (TestProtocol) RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID, addr, conf);
+ // Connect to the server
proxy.ping();
- fail("Interruption did not cause IPC to fail");
- } catch (IOException ioe) {
- if (!ioe.toString().contains("InterruptedException")) {
- throw ioe;
+ // Interrupt self, try another call
+ Thread.currentThread().interrupt();
+ try {
+ proxy.ping();
+ fail("Interruption did not cause IPC to fail");
+ } catch (IOException ioe) {
+ if (!ioe.toString().contains("InterruptedException")) {
+ throw ioe;
+ }
+ // clear interrupt status for future tests
+ Thread.interrupted();
}
- // clear interrupt status for future tests
- Thread.interrupted();
+ } finally {
+ server.stop();
}
}
@@ -862,59 +866,62 @@ public class TestRPC {
);
server.start();
-
- int numConcurrentRPC = 200;
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
- final CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
- final CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
- final AtomicBoolean leaderRunning = new AtomicBoolean(true);
- final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
- Thread leaderThread = null;
-
- for (int i = 0; i < numConcurrentRPC; i++) {
- final int num = i;
- final TestProtocol proxy = (TestProtocol) RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID, addr, conf);
- Thread rpcThread = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- barrier.await();
- while (num == 0 || leaderRunning.get()) {
+ try {
+ int numConcurrentRPC = 200;
+ InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ final CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
+ final CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
+ final AtomicBoolean leaderRunning = new AtomicBoolean(true);
+ final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
+ Thread leaderThread = null;
+
+ for (int i = 0; i < numConcurrentRPC; i++) {
+ final int num = i;
+ final TestProtocol proxy = (TestProtocol) RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID, addr, conf);
+ Thread rpcThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ barrier.await();
+ while (num == 0 || leaderRunning.get()) {
+ proxy.slowPing(false);
+ }
+
proxy.slowPing(false);
+ } catch (Exception e) {
+ if (num == 0) {
+ leaderRunning.set(false);
+ } else {
+ error.set(e);
+ }
+
+ LOG.error(e);
+ } finally {
+ latch.countDown();
}
-
- proxy.slowPing(false);
- } catch (Exception e) {
- if (num == 0) {
- leaderRunning.set(false);
- } else {
- error.set(e);
- }
-
- LOG.error(e);
- } finally {
- latch.countDown();
}
+ });
+ rpcThread.start();
+
+ if (leaderThread == null) {
+ leaderThread = rpcThread;
}
- });
- rpcThread.start();
-
- if (leaderThread == null) {
- leaderThread = rpcThread;
}
+ // let threads get past the barrier
+ Thread.sleep(1000);
+ // stop a single thread
+ while (leaderRunning.get()) {
+ leaderThread.interrupt();
+ }
+
+ latch.await();
+
+ // should not cause any other thread to get an error
+ assertTrue("rpc got exception " + error.get(), error.get() == null);
+ } finally {
+ server.stop();
}
- // let threads get past the barrier
- Thread.sleep(1000);
- // stop a single thread
- while (leaderRunning.get()) {
- leaderThread.interrupt();
- }
-
- latch.await();
-
- // should not cause any other thread to get an error
- assertTrue("rpc got exception " + error.get(), error.get() == null);
}
public static void main(String[] args) throws Exception {
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java Mon Mar 18 11:45:07 2013
@@ -605,7 +605,7 @@ public class TestNetUtils {
@Test
public void testNormalizeHostName() {
List<String> hosts = Arrays.asList(new String[] {"127.0.0.1",
- "localhost", "3w.org", "UnknownHost"});
+ "localhost", "3w.org", "UnknownHost123"});
List<String> normalizedHosts = NetUtils.normalizeHostNames(hosts);
// when ipaddress is normalized, same address is expected in return
assertEquals(normalizedHosts.get(0), hosts.get(0));
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java?rev=1457716&r1=1457715&r2=1457716&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java Mon Mar 18 11:45:07 2013
@@ -162,6 +162,9 @@ public abstract class GenericTestUtils {
private final CountDownLatch waitLatch = new CountDownLatch(1);
private final CountDownLatch resultLatch = new CountDownLatch(1);
+ private final AtomicInteger fireCounter = new AtomicInteger(0);
+ private final AtomicInteger resultCounter = new AtomicInteger(0);
+
// Result fields set after proceed() is called.
private volatile Throwable thrown;
private volatile Object returnValue;
@@ -188,6 +191,7 @@ public abstract class GenericTestUtils {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
LOG.info("DelayAnswer firing fireLatch");
+ fireCounter.getAndIncrement();
fireLatch.countDown();
try {
LOG.info("DelayAnswer waiting on waitLatch");
@@ -208,6 +212,7 @@ public abstract class GenericTestUtils {
thrown = t;
throw t;
} finally {
+ resultCounter.incrementAndGet();
resultLatch.countDown();
}
}
@@ -235,6 +240,14 @@ public abstract class GenericTestUtils {
public Object getReturnValue() {
return returnValue;
}
+
+ public int getFireCount() {
+ return fireCounter.get();
+ }
+
+ public int getResultCount() {
+ return resultCounter.get();
+ }
}
/**