You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/07/06 22:57:23 UTC
[01/48] hadoop git commit: HDFS-7390. Provide JMX metrics per storage
type. (Benoy Antony)
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 18c48592e -> 683755269
HDFS-7390. Provide JMX metrics per storage type. (Benoy Antony)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/130d7d1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/130d7d1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/130d7d1b
Branch: refs/heads/YARN-2928
Commit: 130d7d1b644cf61e295dfbf1654ffb82f1eabe56
Parents: 18c4859
Author: Benoy Antony <be...@apache.org>
Authored: Mon Jun 29 11:00:22 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:00 2015 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 15 +-
.../blockmanagement/BlockStatsMXBean.java | 36 +++++
.../blockmanagement/DatanodeStatistics.java | 6 +
.../blockmanagement/HeartbeatManager.java | 96 +++++++++++-
.../blockmanagement/StorageTypeStats.java | 115 +++++++++++++++
.../blockmanagement/TestBlockStatsMXBean.java | 146 +++++++++++++++++++
6 files changed, 412 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5bd4980..0b60a97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -39,6 +39,8 @@ import java.util.TreeSet;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
+import javax.management.ObjectName;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -85,6 +87,7 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
+import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Daemon;
@@ -94,6 +97,7 @@ import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,7 +105,7 @@ import org.slf4j.LoggerFactory;
* Keeps information related to the blocks stored in the Hadoop cluster.
*/
@InterfaceAudience.Private
-public class BlockManager {
+public class BlockManager implements BlockStatsMXBean {
public static final Logger LOG = LoggerFactory.getLogger(BlockManager.class);
public static final Logger blockLog = NameNode.blockStateChangeLog;
@@ -129,6 +133,7 @@ public class BlockManager {
private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L);
private final long startupDelayBlockDeletionInMs;
private final BlockReportLeaseManager blockReportLeaseManager;
+ private ObjectName mxBeanName;
/** Used by metrics */
public long getPendingReplicationBlocksCount() {
@@ -468,6 +473,7 @@ public class BlockManager {
pendingReplications.start();
datanodeManager.activate(conf);
this.replicationThread.start();
+ mxBeanName = MBeans.register("NameNode", "BlockStats", this);
}
public void close() {
@@ -3944,6 +3950,8 @@ public class BlockManager {
public void shutdown() {
stopReplicationInitializer();
blocksMap.close();
+ MBeans.unregister(mxBeanName);
+ mxBeanName = null;
}
public void clear() {
@@ -3954,4 +3962,9 @@ public class BlockManager {
public BlockReportLeaseManager getBlockReportLeaseManager() {
return blockReportLeaseManager;
}
+
+ @Override // BlockStatsMXBean
+ public Map<StorageType, StorageTypeStats> getStorageTypeStats() {
+ return datanodeManager.getDatanodeStatistics().getStorageTypeStats();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
new file mode 100644
index 0000000..f22c537
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
+
+/**
+ * This is an interface used to retrieve statistic information related to
+ * block management.
+ */
+public interface BlockStatsMXBean {
+
+ /**
+ * The statistics of storage types.
+ *
+ * @return get storage statistics per storage type
+ */
+ Map<StorageType, StorageTypeStats> getStorageTypeStats();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
index c9bc3e5..33eca2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
/** Datanode statistics */
@@ -71,4 +74,7 @@ public interface DatanodeStatistics {
/** @return the expired heartbeats */
public int getExpiredHeartbeats();
+
+ /** @return Storage Tier statistics*/
+ Map<StorageType, StorageTypeStats> getStorageTypeStats();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 9017fe1..cc9365d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -18,9 +18,15 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -189,6 +195,11 @@ class HeartbeatManager implements DatanodeStatistics {
return stats.expiredHeartbeats;
}
+ @Override
+ public Map<StorageType, StorageTypeStats> getStorageTypeStats() {
+ return stats.statsMap.get();
+ }
+
synchronized void register(final DatanodeDescriptor d) {
if (!d.isAlive) {
addDatanode(d);
@@ -393,6 +404,9 @@ class HeartbeatManager implements DatanodeStatistics {
* For decommissioning/decommissioned nodes, only used capacity is counted.
*/
private static class Stats {
+
+ private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
+
private long capacityTotal = 0L;
private long capacityUsed = 0L;
private long capacityRemaining = 0L;
@@ -420,6 +434,14 @@ class HeartbeatManager implements DatanodeStatistics {
}
cacheCapacity += node.getCacheCapacity();
cacheUsed += node.getCacheUsed();
+ Set<StorageType> storageTypes = new HashSet<>();
+ for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) {
+ statsMap.addStorage(storageInfo, node);
+ storageTypes.add(storageInfo.getStorageType());
+ }
+ for (StorageType storageType : storageTypes) {
+ statsMap.addNode(storageType, node);
+ }
}
private void subtract(final DatanodeDescriptor node) {
@@ -436,6 +458,14 @@ class HeartbeatManager implements DatanodeStatistics {
}
cacheCapacity -= node.getCacheCapacity();
cacheUsed -= node.getCacheUsed();
+ Set<StorageType> storageTypes = new HashSet<>();
+ for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) {
+ statsMap.subtractStorage(storageInfo, node);
+ storageTypes.add(storageInfo.getStorageType());
+ }
+ for (StorageType storageType : storageTypes) {
+ statsMap.subtractNode(storageType, node);
+ }
}
/** Increment expired heartbeat counter. */
@@ -443,5 +473,69 @@ class HeartbeatManager implements DatanodeStatistics {
expiredHeartbeats++;
}
}
-}
+ /** StorageType specific statistics.
+ * For decommissioning/decommissioned nodes, only used capacity is counted.
+ */
+
+ static final class StorageTypeStatsMap {
+
+ private Map<StorageType, StorageTypeStats> storageTypeStatsMap =
+ new IdentityHashMap<>();
+
+ private StorageTypeStatsMap() {}
+
+ private StorageTypeStatsMap(StorageTypeStatsMap other) {
+ storageTypeStatsMap =
+ new IdentityHashMap<>(other.storageTypeStatsMap);
+ for (Map.Entry<StorageType, StorageTypeStats> entry :
+ storageTypeStatsMap.entrySet()) {
+ entry.setValue(new StorageTypeStats(entry.getValue()));
+ }
+ }
+
+ private Map<StorageType, StorageTypeStats> get() {
+ return Collections.unmodifiableMap(storageTypeStatsMap);
+ }
+
+ private void addNode(StorageType storageType,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(storageType);
+ if (storageTypeStats == null) {
+ storageTypeStats = new StorageTypeStats();
+ storageTypeStatsMap.put(storageType, storageTypeStats);
+ }
+ storageTypeStats.addNode(node);
+ }
+
+ private void addStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(info.getStorageType());
+ if (storageTypeStats == null) {
+ storageTypeStats = new StorageTypeStats();
+ storageTypeStatsMap.put(info.getStorageType(), storageTypeStats);
+ }
+ storageTypeStats.addStorage(info, node);
+ }
+
+ private void subtractStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(info.getStorageType());
+ if (storageTypeStats != null) {
+ storageTypeStats.subtractStorage(info, node);
+ }
+ }
+
+ private void subtractNode(StorageType storageType,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(storageType);
+ if (storageTypeStats != null) {
+ storageTypeStats.subtractNode(node);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
new file mode 100644
index 0000000..45dcc8d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.beans.ConstructorProperties;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Statistics per StorageType.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class StorageTypeStats {
+ private long capacityTotal = 0L;
+ private long capacityUsed = 0L;
+ private long capacityRemaining = 0L;
+ private long blockPoolUsed = 0L;
+ private int nodesInService = 0;
+
+ @ConstructorProperties({"capacityTotal",
+ "capacityUsed", "capacityRemaining", "blockPoolUsed", "nodesInService"})
+ public StorageTypeStats(long capacityTotal, long capacityUsed,
+ long capacityRemaining, long blockPoolUsed, int nodesInService) {
+ this.capacityTotal = capacityTotal;
+ this.capacityUsed = capacityUsed;
+ this.capacityRemaining = capacityRemaining;
+ this.blockPoolUsed = blockPoolUsed;
+ this.nodesInService = nodesInService;
+ }
+
+ public long getCapacityTotal() {
+ return capacityTotal;
+ }
+
+ public long getCapacityUsed() {
+ return capacityUsed;
+ }
+
+ public long getCapacityRemaining() {
+ return capacityRemaining;
+ }
+
+ public long getBlockPoolUsed() {
+ return blockPoolUsed;
+ }
+
+ public int getNodesInService() {
+ return nodesInService;
+ }
+
+ StorageTypeStats() {}
+
+ StorageTypeStats(StorageTypeStats other) {
+ capacityTotal = other.capacityTotal;
+ capacityUsed = other.capacityUsed;
+ capacityRemaining = other.capacityRemaining;
+ blockPoolUsed = other.blockPoolUsed;
+ nodesInService = other.nodesInService;
+ }
+
+ void addStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ capacityUsed += info.getDfsUsed();
+ blockPoolUsed += info.getBlockPoolUsed();
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ capacityTotal += info.getCapacity();
+ capacityRemaining += info.getRemaining();
+ } else {
+ capacityTotal += info.getDfsUsed();
+ }
+ }
+
+ void addNode(final DatanodeDescriptor node) {
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ nodesInService++;
+ }
+ }
+
+ void subtractStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ capacityUsed -= info.getDfsUsed();
+ blockPoolUsed -= info.getBlockPoolUsed();
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ capacityTotal -= info.getCapacity();
+ capacityRemaining -= info.getRemaining();
+ } else {
+ capacityTotal -= info.getDfsUsed();
+ }
+ }
+
+ void subtractNode(final DatanodeDescriptor node) {
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ nodesInService--;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
new file mode 100644
index 0000000..43d983d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
+
+/**
+ * Class for testing {@link BlockStatsMXBean} implementation
+ */
+public class TestBlockStatsMXBean {
+
+ private MiniDFSCluster cluster;
+
+ @Before
+ public void setup() throws IOException {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ cluster = null;
+ StorageType[][] types = new StorageType[6][];
+ for (int i=0; i<3; i++) {
+ types[i] = new StorageType[] {StorageType.RAM_DISK, StorageType.DISK};
+ }
+ for (int i=3; i< 5; i++) {
+ types[i] = new StorageType[] {StorageType.RAM_DISK, StorageType.ARCHIVE};
+ }
+ types[5] = new StorageType[] {StorageType.RAM_DISK, StorageType.ARCHIVE,
+ StorageType.ARCHIVE};
+
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(6).
+ storageTypes(types).storagesPerDatanode(3).build();
+ cluster.waitActive();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testStorageTypeStats() throws Exception {
+ Map<StorageType, StorageTypeStats> storageTypeStatsMap =
+ cluster.getNamesystem().getBlockManager().getStorageTypeStats();
+ assertTrue(storageTypeStatsMap.containsKey(StorageType.RAM_DISK));
+ assertTrue(storageTypeStatsMap.containsKey(StorageType.DISK));
+ assertTrue(storageTypeStatsMap.containsKey(StorageType.ARCHIVE));
+
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(StorageType.RAM_DISK);
+ assertEquals(6, storageTypeStats.getNodesInService());
+
+ storageTypeStats = storageTypeStatsMap.get(StorageType.DISK);
+ assertEquals(3, storageTypeStats.getNodesInService());
+
+ storageTypeStats = storageTypeStatsMap.get(StorageType.ARCHIVE);
+ assertEquals(3, storageTypeStats.getNodesInService());
+ }
+
+ protected static String readOutput(URL url) throws IOException {
+ StringBuilder out = new StringBuilder();
+ InputStream in = url.openConnection().getInputStream();
+ byte[] buffer = new byte[64 * 1024];
+ int len = in.read(buffer);
+ while (len > 0) {
+ out.append(new String(buffer, 0, len));
+ len = in.read(buffer);
+ }
+ return out.toString();
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testStorageTypeStatsJMX() throws Exception {
+ URL baseUrl = new URL (cluster.getHttpUri(0));
+ String result = readOutput(new URL(baseUrl, "/jmx"));
+ System.out.println(result);
+
+ Map<String, Object> stat = (Map<String, Object>) JSON.parse(result);
+ Object[] beans =(Object[]) stat.get("beans");
+ Map<String, Object> blockStats = null;
+ for (Object bean : beans) {
+ Map<String, Object> map = (Map<String, Object>) bean;
+ if (map.get("name").equals("Hadoop:service=NameNode,name=BlockStats")) {
+ blockStats = map;
+ }
+ }
+ assertNotNull(blockStats);
+ Object[] storageTypeStatsList =
+ (Object[])blockStats.get("StorageTypeStats");
+ assertNotNull(storageTypeStatsList);
+ assertEquals (3, storageTypeStatsList.length);
+
+ Set<String> typesPresent = new HashSet<> ();
+ for (Object obj : storageTypeStatsList) {
+ Map<String, Object> entry = (Map<String, Object>)obj;
+ String storageType = (String)entry.get("key");
+ Map<String,Object> storageTypeStats = (Map<String,Object>)entry.get("value");
+ typesPresent.add(storageType);
+ if (storageType.equals("ARCHIVE") || storageType.equals("DISK") ) {
+ assertEquals(3l, storageTypeStats.get("nodesInService"));
+ } else if (storageType.equals("RAM_DISK")) {
+ assertEquals(6l, storageTypeStats.get("nodesInService"));
+ }
+ else {
+ fail();
+ }
+ }
+
+ assertTrue(typesPresent.contains("ARCHIVE"));
+ assertTrue(typesPresent.contains("DISK"));
+ assertTrue(typesPresent.contains("RAM_DISK"));
+ }
+}
[13/48] hadoop git commit: HADOOP-12154. FileSystem#getUsed() returns
the file length only from root '/' (Contributed by J.Andreina)
Posted by zj...@apache.org.
HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9ae815f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9ae815f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9ae815f
Branch: refs/heads/YARN-2928
Commit: c9ae815f6a67e8f3150b4dd4f209a63d7da97b89
Parents: a943699
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:25:20 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/fs/FileSystem.java | 6 ++---
.../org/apache/hadoop/hdfs/TestDFSShell.java | 26 ++++++++++++++++++++
3 files changed, 32 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 50192ae..2cf9509 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -899,6 +899,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12089. StorageException complaining " no lease ID" when updating
FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
+ HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 1d7bc87..c98074a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2085,9 +2085,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/** Return the total size of all files in the filesystem.*/
public long getUsed() throws IOException{
long used = 0;
- FileStatus[] files = listStatus(new Path("/"));
- for(FileStatus file:files){
- used += file.getLen();
+ RemoteIterator<LocatedFileStatus> files = listFiles(new Path("/"), true);
+ while (files.hasNext()) {
+ used += files.next().getLen();
}
return used;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 2df31c4..1386124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -913,6 +913,32 @@ public class TestDFSShell {
cluster.shutdown();
}
}
+
+ @Test(timeout = 30000)
+ public void testTotalSizeOfAllFiles() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ FileSystem fs = cluster.getFileSystem();
+ // create file under root
+ FSDataOutputStream File1 = fs.create(new Path("/File1"));
+ File1.write("hi".getBytes());
+ File1.close();
+ // create file under sub-folder
+ FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2"));
+ File2.write("hi".getBytes());
+ File2.close();
+ // getUsed() should return total length of all the files in Filesystem
+ assertEquals(4, fs.getUsed());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+ }
+
private static void runCount(String path, long dirs, long files, FsShell shell
) throws IOException {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
[19/48] hadoop git commit: MAPREDUCE-6384. Add the last reporting
reducer info for too many fetch failure diagnostics. Contributed by Chang Li
Posted by zj...@apache.org.
MAPREDUCE-6384. Add the last reporting reducer info for too many fetch failure diagnostics. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9465dc4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9465dc4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9465dc4c
Branch: refs/heads/YARN-2928
Commit: 9465dc4ce7f5e07fb868f61f43357b676b2f5e84
Parents: c68da70
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Jun 30 21:22:30 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../event/JobTaskAttemptFetchFailureEvent.java | 9 +-
.../TaskAttemptTooManyFetchFailureEvent.java | 50 +++++++
.../mapreduce/v2/app/job/impl/JobImpl.java | 7 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 15 +-
.../mapreduce/v2/app/TestFetchFailure.java | 31 ++--
.../v2/app/job/impl/TestTaskAttempt.java | 143 ++++++++++---------
7 files changed, 168 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index eec131c..974af61 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -388,6 +388,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6408. Queue name and user name should be printed on the job page.
(Siqi Li via gera)
+ MAPREDUCE-6384. Add the last reporting reducer info for too many fetch
+ failure diagnostics (Chang Li via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
index 37e2034..787711c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
@@ -28,13 +28,15 @@ public class JobTaskAttemptFetchFailureEvent extends JobEvent {
private final TaskAttemptId reduce;
private final List<TaskAttemptId> maps;
+ private final String hostname;
public JobTaskAttemptFetchFailureEvent(TaskAttemptId reduce,
- List<TaskAttemptId> maps) {
- super(reduce.getTaskId().getJobId(),
+ List<TaskAttemptId> maps, String host) {
+ super(reduce.getTaskId().getJobId(),
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE);
this.reduce = reduce;
this.maps = maps;
+ this.hostname = host;
}
public List<TaskAttemptId> getMaps() {
@@ -45,4 +47,7 @@ public class JobTaskAttemptFetchFailureEvent extends JobEvent {
return reduce;
}
+ public String getHost() {
+ return hostname;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
new file mode 100644
index 0000000..662e712
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
@@ -0,0 +1,50 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+/**
+ * TaskAttemptTooManyFetchFailureEvent is used for TA_TOO_MANY_FETCH_FAILURE.
+ */
+public class TaskAttemptTooManyFetchFailureEvent extends TaskAttemptEvent {
+ private TaskAttemptId reduceID;
+ private String reduceHostname;
+
+ /**
+ * Create a new TaskAttemptTooManyFetchFailureEvent.
+ * @param attemptId the id of the mapper task attempt
+ * @param reduceId the id of the reporting reduce task attempt.
+ * @param reduceHost the hostname of the reporting reduce task attempt.
+ */
+ public TaskAttemptTooManyFetchFailureEvent(TaskAttemptId attemptId,
+ TaskAttemptId reduceId, String reduceHost) {
+ super(attemptId, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE);
+ this.reduceID = reduceId;
+ this.reduceHostname = reduceHost;
+ }
+
+ public TaskAttemptId getReduceId() {
+ return reduceID;
+ }
+
+ public String getReduceHost() {
+ return reduceHostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 6e9f13c..2c48019 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -103,9 +103,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskRecoverEvent;
@@ -1914,8 +1913,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
&& failureRate >= job.getMaxAllowedFetchFailuresFraction()) {
LOG.info("Too many fetch-failures for output of task attempt: " +
mapId + " ... raising fetch failure to map");
- job.eventHandler.handle(new TaskAttemptEvent(mapId,
- TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+ job.eventHandler.handle(new TaskAttemptTooManyFetchFailureEvent(mapId,
+ fetchfailureEvent.getReduce(), fetchfailureEvent.getHost()));
job.fetchFailuresMapping.remove(mapId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3055a25..3fa42fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptRecoverEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
@@ -1916,12 +1917,17 @@ public abstract class TaskAttemptImpl implements
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+ TaskAttemptTooManyFetchFailureEvent fetchFailureEvent =
+ (TaskAttemptTooManyFetchFailureEvent) event;
// too many fetch failure can only happen for map tasks
Preconditions
.checkArgument(taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP);
//add to diagnostic
- taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
-
+ taskAttempt.addDiagnosticInfo("Too many fetch failures."
+ + " Failing the attempt. Last failure reported by " +
+ fetchFailureEvent.getReduceId() +
+ " from host " + fetchFailureEvent.getReduceHost());
+
if (taskAttempt.getLaunchTime() != 0) {
taskAttempt.eventHandler
.handle(createJobCounterUpdateEventTAFailed(taskAttempt, true));
@@ -2225,8 +2231,11 @@ public abstract class TaskAttemptImpl implements
//this only will happen in reduce attempt type
if (taskAttempt.reportedStatus.fetchFailedMaps != null &&
taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) {
+ String hostname = taskAttempt.container == null ? "UNKNOWN"
+ : taskAttempt.container.getNodeId().getHost();
taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent(
- taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps));
+ taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps,
+ hostname));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
index 4e4e2e7..8d25079 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
@@ -94,10 +94,10 @@ public class TestFetchFailure {
app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);
//send 3 fetch failures from reduce to trigger map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
-
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+
//wait for map Task state move back to RUNNING
app.waitForState(mapTask, TaskState.RUNNING);
@@ -215,9 +215,9 @@ public class TestFetchFailure {
app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);
//send 3 fetch failures from reduce to trigger map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
//wait for map Task state move back to RUNNING
app.waitForState(mapTask, TaskState.RUNNING);
@@ -324,8 +324,8 @@ public class TestFetchFailure {
updateStatus(app, reduceAttempt3, Phase.SHUFFLE);
//send 2 fetch failures from reduce to prepare for map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host1");
+ sendFetchFailure(app, reduceAttempt2, mapAttempt1, "host2");
//We should not re-launch the map task yet
assertEquals(TaskState.SUCCEEDED, mapTask.getState());
@@ -333,7 +333,7 @@ public class TestFetchFailure {
updateStatus(app, reduceAttempt3, Phase.REDUCE);
//send 3rd fetch failures from reduce to trigger map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
+ sendFetchFailure(app, reduceAttempt3, mapAttempt1, "host3");
//wait for map Task state move back to RUNNING
app.waitForState(mapTask, TaskState.RUNNING);
@@ -342,6 +342,11 @@ public class TestFetchFailure {
Assert.assertEquals("Map TaskAttempt state not correct",
TaskAttemptState.FAILED, mapAttempt1.getState());
+ Assert.assertEquals(mapAttempt1.getDiagnostics().get(0),
+ "Too many fetch failures. Failing the attempt. "
+ + "Last failure reported by "
+ + reduceAttempt3.getID().toString() + " from host host3");
+
Assert.assertEquals("Num attempts in Map Task not correct",
2, mapTask.getAttempts().size());
@@ -410,7 +415,6 @@ public class TestFetchFailure {
Assert.assertEquals("Unexpected map event", convertedEvents[2],
mapEvents[0]);
}
-
private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) {
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus();
@@ -430,11 +434,12 @@ public class TestFetchFailure {
}
private void sendFetchFailure(MRApp app, TaskAttempt reduceAttempt,
- TaskAttempt mapAttempt) {
+ TaskAttempt mapAttempt, String hostname) {
app.getContext().getEventHandler().handle(
new JobTaskAttemptFetchFailureEvent(
reduceAttempt.getID(),
- Arrays.asList(new TaskAttemptId[] {mapAttempt.getID()})));
+ Arrays.asList(new TaskAttemptId[] {mapAttempt.getID()}),
+ hostname));
}
static class MRAppWithHistory extends MRApp {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 79b88d8..a88a935 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
@@ -507,6 +508,9 @@ public class TestTaskAttempt{
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ TaskId reduceTaskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reduceTaskId, 0);
Path jobFile = mock(Path.class);
MockEventHandler eventHandler = new MockEventHandler();
@@ -554,8 +558,8 @@ public class TestTaskAttempt{
assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
TaskAttemptState.SUCCEEDED);
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId,
+ reduceTAId, "Host"));
assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
TaskAttemptState.FAILED);
taImpl.handle(new TaskAttemptEvent(attemptId,
@@ -735,72 +739,75 @@ public class TestTaskAttempt{
@Test
public void testFetchFailureAttemptFinishTime() throws Exception{
- ApplicationId appId = ApplicationId.newInstance(1, 2);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 0);
- JobId jobId = MRBuilderUtils.newJobId(appId, 1);
- TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
- TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
- Path jobFile = mock(Path.class);
-
- MockEventHandler eventHandler = new MockEventHandler();
- TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
-
- JobConf jobConf = new JobConf();
- jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
- jobConf.setBoolean("fs.file.impl.disable.cache", true);
- jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
- jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-
- TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
- when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-
- AppContext appCtx = mock(AppContext.class);
- ClusterInfo clusterInfo = mock(ClusterInfo.class);
- when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
-
- TaskAttemptImpl taImpl =
- new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
- splits, jobConf, taListener,mock(Token.class), new Credentials(),
- new SystemClock(), appCtx);
-
- NodeId nid = NodeId.newInstance("127.0.0.1", 0);
- ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
- Container container = mock(Container.class);
- when(container.getId()).thenReturn(contId);
- when(container.getNodeId()).thenReturn(nid);
- when(container.getNodeHttpAddress()).thenReturn("localhost:0");
-
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_SCHEDULE));
- taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
- container, mock(Map.class)));
- taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_DONE));
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_CONTAINER_COMPLETED));
-
- assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
- TaskAttemptState.SUCCEEDED);
-
- assertTrue("Task Attempt finish time is not greater than 0",
- taImpl.getFinishTime() > 0);
-
- Long finishTime = taImpl.getFinishTime();
- Thread.sleep(5);
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
-
- assertEquals("Task attempt is not in Too Many Fetch Failure state",
- taImpl.getState(), TaskAttemptState.FAILED);
-
- assertEquals("After TA_TOO_MANY_FETCH_FAILURE,"
- + " Task attempt finish time is not the same ",
- finishTime, Long.valueOf(taImpl.getFinishTime()));
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+ Path jobFile = mock(Path.class);
+
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(
+ new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+ TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+ when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+ AppContext appCtx = mock(AppContext.class);
+ ClusterInfo clusterInfo = mock(ClusterInfo.class);
+ when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
+ setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ splits, jobConf, taListener,mock(Token.class), new Credentials(),
+ new SystemClock(), appCtx);
+
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_SCHEDULE));
+ taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+ container, mock(Map.class)));
+ taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_DONE));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+
+ assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
+ TaskAttemptState.SUCCEEDED);
+
+ assertTrue("Task Attempt finish time is not greater than 0",
+ taImpl.getFinishTime() > 0);
+
+ Long finishTime = taImpl.getFinishTime();
+ Thread.sleep(5);
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId,
+ reduceTAId, "Host"));
+
+ assertEquals("Task attempt is not in Too Many Fetch Failure state",
+ taImpl.getState(), TaskAttemptState.FAILED);
+
+ assertEquals("After TA_TOO_MANY_FETCH_FAILURE,"
+ + " Task attempt finish time is not the same ",
+ finishTime, Long.valueOf(taImpl.getFinishTime()));
}
@Test
[37/48] hadoop git commit: HDFS-8666. Speedup the TestMover test.
Contributed by Walter Su.
Posted by zj...@apache.org.
HDFS-8666. Speedup the TestMover test. Contributed by Walter Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2820f1f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2820f1f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2820f1f4
Branch: refs/heads/YARN-2928
Commit: 2820f1f4b7c4a77b9d6481e3c13611979581fed7
Parents: e556ee0
Author: Jing Zhao <ji...@apache.org>
Authored: Wed Jul 1 14:59:50 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop/hdfs/server/mover/TestMover.java | 19 +++++++++++++++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2820f1f4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3736932..ec37542 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -694,6 +694,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
via Colin P. McCabe)
+ HDFS-8666. Speedup the TestMover tests. (Walter Su via jing9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2820f1f4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 49e2b23..899b5c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DBlock;
import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
+import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.server.mover.Mover.MLocation;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.test.GenericTestUtils;
@@ -46,6 +47,21 @@ import org.junit.Assert;
import org.junit.Test;
public class TestMover {
+
+ static final int DEFAULT_BLOCK_SIZE = 100;
+
+ static {
+ TestBalancer.initTestSetup();
+ }
+
+ static void initConf(Configuration conf) {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
+ }
+
static Mover newMover(Configuration conf) throws IOException {
final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
Assert.assertEquals(1, namenodes.size());
@@ -97,6 +113,7 @@ public class TestMover {
@Test
public void testScheduleBlockWithinSameNode() throws Exception {
final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3)
.storageTypes(
@@ -285,6 +302,7 @@ public class TestMover {
public void testTwoReplicaSameStorageTypeShouldNotSelect() throws Exception {
// HDFS-8147
final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3)
.storageTypes(
@@ -361,6 +379,7 @@ public class TestMover {
public void testMoverFailedRetry() throws Exception {
// HDFS-8147
final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2");
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3)
[46/48] hadoop git commit: HADOOP-12186. ActiveStandbyElector
shouldn't call monitorLockNodeAsync multiple times (Contributed by zhihai xu)
Posted by zj...@apache.org.
HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync multiple times (Contributed by zhihai xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7182a03f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7182a03f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7182a03f
Branch: refs/heads/YARN-2928
Commit: 7182a03fdc23f83912b41d71f09f906f9594b0ba
Parents: b66c6cb
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jul 6 15:39:43 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/ha/ActiveStandbyElector.java | 20 +++++++++++--
.../hadoop/ha/TestActiveStandbyElector.java | 31 ++++++++++++++++++++
3 files changed, 51 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7182a03f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 312a996..1d737e5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -924,6 +924,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
name using reflection. (Lei (Eddy) Xu)
+ HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync
+ multiple times (zhihai xu via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7182a03f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index e520a16..e458181 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -173,7 +173,9 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
private Lock sessionReestablishLockForTests = new ReentrantLock();
private boolean wantToBeInElection;
-
+ private boolean monitorLockNodePending = false;
+ private ZooKeeper monitorLockNodeClient;
+
/**
* Create a new ActiveStandbyElector object <br/>
* The elector is created by providing to it the Zookeeper configuration, the
@@ -468,7 +470,8 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
public synchronized void processResult(int rc, String path, Object ctx,
Stat stat) {
if (isStaleClient(ctx)) return;
-
+ monitorLockNodePending = false;
+
assert wantToBeInElection :
"Got a StatNode result after quitting election";
@@ -744,6 +747,11 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
return state;
}
+ @VisibleForTesting
+ synchronized boolean isMonitorLockNodePending() {
+ return monitorLockNodePending;
+ }
+
private boolean reEstablishSession() {
int connectionRetryCount = 0;
boolean success = false;
@@ -949,7 +957,13 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
}
private void monitorLockNodeAsync() {
- zkClient.exists(zkLockFilePath,
+ if (monitorLockNodePending && monitorLockNodeClient == zkClient) {
+ LOG.info("Ignore duplicate monitor lock-node request.");
+ return;
+ }
+ monitorLockNodePending = true;
+ monitorLockNodeClient = zkClient;
+ zkClient.exists(zkLockFilePath,
watcher, this,
zkClient);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7182a03f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
index 2e578e2..83a3a4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
@@ -452,6 +452,10 @@ public class TestActiveStandbyElector {
Event.KeeperState.SyncConnected);
elector.processWatchEvent(mockZK, mockEvent);
verifyExistCall(1);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+ elector.processResult(Code.SESSIONEXPIRED.intValue(), ZK_LOCK_NAME,
+ mockZK, new Stat());
+ Assert.assertFalse(elector.isMonitorLockNodePending());
// session expired should enter safe mode and initiate re-election
// re-election checked via checking re-creation of new zookeeper and
@@ -495,6 +499,13 @@ public class TestActiveStandbyElector {
ZK_LOCK_NAME);
Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
verifyExistCall(1);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+
+ Stat stat = new Stat();
+ stat.setEphemeralOwner(0L);
+ Mockito.when(mockZK.getSessionId()).thenReturn(1L);
+ elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+ Assert.assertFalse(elector.isMonitorLockNodePending());
WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
@@ -504,12 +515,18 @@ public class TestActiveStandbyElector {
Event.EventType.NodeDataChanged);
elector.processWatchEvent(mockZK, mockEvent);
verifyExistCall(2);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+ elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+ Assert.assertFalse(elector.isMonitorLockNodePending());
// monitoring should be setup again after event is received
Mockito.when(mockEvent.getType()).thenReturn(
Event.EventType.NodeChildrenChanged);
elector.processWatchEvent(mockZK, mockEvent);
verifyExistCall(3);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+ elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+ Assert.assertFalse(elector.isMonitorLockNodePending());
// lock node deletion when in standby mode should create znode again
// successful znode creation enters active state and sets monitor
@@ -524,6 +541,10 @@ public class TestActiveStandbyElector {
ZK_LOCK_NAME);
Mockito.verify(mockApp, Mockito.times(1)).becomeActive();
verifyExistCall(4);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+ stat.setEphemeralOwner(1L);
+ elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+ Assert.assertFalse(elector.isMonitorLockNodePending());
// lock node deletion in active mode should enter neutral mode and create
// znode again successful znode creation enters active state and sets
@@ -538,6 +559,9 @@ public class TestActiveStandbyElector {
ZK_LOCK_NAME);
Mockito.verify(mockApp, Mockito.times(2)).becomeActive();
verifyExistCall(5);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+ elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+ Assert.assertFalse(elector.isMonitorLockNodePending());
// bad path name results in fatal error
Mockito.when(mockEvent.getPath()).thenReturn(null);
@@ -570,6 +594,13 @@ public class TestActiveStandbyElector {
ZK_LOCK_NAME);
Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
verifyExistCall(1);
+ Assert.assertTrue(elector.isMonitorLockNodePending());
+
+ Stat stat = new Stat();
+ stat.setEphemeralOwner(0L);
+ Mockito.when(mockZK.getSessionId()).thenReturn(1L);
+ elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+ Assert.assertFalse(elector.isMonitorLockNodePending());
WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
[45/48] hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes
wrong "base" parameter to getMapOutputInfo if mapId is not in the cache.
Contributed by zhihai xu.
Posted by zj...@apache.org.
MAPREDUCE-6425. ShuffleHandler passes wrong "base" parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b66c6cb8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b66c6cb8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b66c6cb8
Branch: refs/heads/YARN-2928
Commit: b66c6cb8be517deb7c87bf58d23dc620a63f9bd1
Parents: ccfc787
Author: Devaraj K <de...@apache.org>
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../apache/hadoop/mapred/ShuffleHandler.java | 3 +-
.../hadoop/mapred/TestShuffleHandler.java | 101 +++++++++++++++++++
3 files changed, 106 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b66c6cb8/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 68ce49b..90208ec 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -566,6 +566,9 @@ Release 2.7.2 - UNRELEASED
BUG FIXES
+ MAPREDUCE-6425. ShuffleHandler passes wrong "base" parameter to getMapOutputInfo
+ if mapId is not in the cache. (zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b66c6cb8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
try {
MapOutputInfo info = mapOutputInfoMap.get(mapId);
if (info == null) {
- info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+ info = getMapOutputInfo(outputBasePathStr + mapId,
+ mapId, reduceId, user);
}
lastMap =
sendMapOutput(ctx, ch, user, mapId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b66c6cb8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 7053653..746071f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
Assert.assertTrue((new String(byteArr)).contains(message));
} finally {
shuffleHandler.stop();
+ FileUtil.fullyDelete(absLogDir);
}
}
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
conn.disconnect();
return rc;
}
+
+ @Test(timeout = 100000)
+ public void testGetMapOutputInfo() throws Exception {
+ final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
+ Configuration conf = new Configuration();
+ conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+ conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+ File absLogDir = new File("target", TestShuffleHandler.class.
+ getSimpleName() + "LocDir").getAbsoluteFile();
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ ApplicationId appId = ApplicationId.newInstance(12345, 1);
+ String appAttemptId = "attempt_12345_1_m_1_0";
+ String user = "randomUser";
+ String reducerId = "0";
+ List<File> fileMap = new ArrayList<File>();
+ createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId,
+ conf, fileMap);
+ ShuffleHandler shuffleHandler = new ShuffleHandler() {
+ @Override
+ protected Shuffle getShuffle(Configuration conf) {
+ // replace the shuffle handler with one stubbed for testing
+ return new Shuffle(conf) {
+ @Override
+ protected void populateHeaders(List<String> mapIds,
+ String outputBaseStr, String user, int reduce,
+ HttpRequest request, HttpResponse response,
+ boolean keepAliveParam, Map<String, MapOutputInfo> infoMap)
+ throws IOException {
+ // Only set response headers and skip everything else
+ // send some dummy value for content-length
+ super.setResponseHeaders(response, keepAliveParam, 100);
+ }
+ @Override
+ protected void verifyRequest(String appid,
+ ChannelHandlerContext ctx, HttpRequest request,
+ HttpResponse response, URL requestUri) throws IOException {
+ // Do nothing.
+ }
+ @Override
+ protected void sendError(ChannelHandlerContext ctx, String message,
+ HttpResponseStatus status) {
+ if (failures.size() == 0) {
+ failures.add(new Error(message));
+ ctx.getChannel().close();
+ }
+ }
+ @Override
+ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
+ Channel ch, String user, String mapId, int reduce,
+ MapOutputInfo info) throws IOException {
+ // send a shuffle header
+ ShuffleHeader header =
+ new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ header.write(dob);
+ return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+ }
+ };
+ }
+ };
+ shuffleHandler.init(conf);
+ try {
+ shuffleHandler.start();
+ DataOutputBuffer outputBuffer = new DataOutputBuffer();
+ outputBuffer.reset();
+ Token<JobTokenIdentifier> jt =
+ new Token<JobTokenIdentifier>("identifier".getBytes(),
+ "password".getBytes(), new Text(user), new Text("shuffleService"));
+ jt.write(outputBuffer);
+ shuffleHandler
+ .initializeApplication(new ApplicationInitializationContext(user,
+ appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
+ outputBuffer.getLength())));
+ URL url =
+ new URL(
+ "http://127.0.0.1:"
+ + shuffleHandler.getConfig().get(
+ ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ + "/mapOutput?job=job_12345_0001&reduce=" + reducerId
+ + "&map=attempt_12345_1_m_1_0");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+ conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+ conn.connect();
+ try {
+ DataInputStream is = new DataInputStream(conn.getInputStream());
+ ShuffleHeader header = new ShuffleHeader();
+ header.readFields(is);
+ is.close();
+ } catch (EOFException e) {
+ // ignore
+ }
+ Assert.assertEquals(failures.size(), 0);
+ } finally {
+ shuffleHandler.stop();
+ FileUtil.fullyDelete(absLogDir);
+ }
+ }
}
[43/48] hadoop git commit: HDFS-8703. Merge refactor of
DFSInputStream from ErasureCoding branch (Contributed by Vinayakumar B)
Posted by zj...@apache.org.
HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/825dbb82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/825dbb82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/825dbb82
Branch: refs/heads/YARN-2928
Commit: 825dbb8201a2b3a35779dd802490541d426581f0
Parents: ec285b3
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jul 2 16:11:50 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 233 +++++++++++++------
.../hadoop/hdfs/TestDFSClientRetries.java | 2 +-
4 files changed, 169 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ec37542..7b96c56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -696,6 +696,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8666. Speedup the TestMover tests. (Walter Su via jing9)
+ HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch
+ (vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f4ceab3..4923a50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1181,7 +1181,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
// Get block info from namenode
TraceScope scope = getPathTraceScope("newDFSInputStream", src);
try {
- return new DFSInputStream(this, src, verifyChecksum);
+ return new DFSInputStream(this, src, verifyChecksum, null);
} finally {
scope.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 6563d7b..7f3722f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -44,6 +44,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.ByteBufferReadable;
@@ -94,35 +95,35 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
@VisibleForTesting
public static boolean tcpReadsDisabledForTesting = false;
private long hedgedReadOpsLoopNumForTesting = 0;
- private final DFSClient dfsClient;
- private AtomicBoolean closed = new AtomicBoolean(false);
- private final String src;
- private final boolean verifyChecksum;
+ protected final DFSClient dfsClient;
+ protected AtomicBoolean closed = new AtomicBoolean(false);
+ protected final String src;
+ protected final boolean verifyChecksum;
// state by stateful read only:
// (protected by lock on this)
/////
private DatanodeInfo currentNode = null;
- private LocatedBlock currentLocatedBlock = null;
- private long pos = 0;
- private long blockEnd = -1;
+ protected LocatedBlock currentLocatedBlock = null;
+ protected long pos = 0;
+ protected long blockEnd = -1;
private BlockReader blockReader = null;
////
// state shared by stateful and positional read:
// (protected by lock on infoLock)
////
- private LocatedBlocks locatedBlocks = null;
+ protected LocatedBlocks locatedBlocks = null;
private long lastBlockBeingWrittenLength = 0;
private FileEncryptionInfo fileEncryptionInfo = null;
- private CachingStrategy cachingStrategy;
+ protected CachingStrategy cachingStrategy;
////
- private final ReadStatistics readStatistics = new ReadStatistics();
+ protected final ReadStatistics readStatistics = new ReadStatistics();
// lock for state shared between read and pread
// Note: Never acquire a lock on <this> with this lock held to avoid deadlocks
// (it's OK to acquire this lock when the lock on <this> is held)
- private final Object infoLock = new Object();
+ protected final Object infoLock = new Object();
/**
* Track the ByteBuffers that we have handed out to readers.
@@ -239,7 +240,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* back to the namenode to get a new list of block locations, and is
* capped at maxBlockAcquireFailures
*/
- private int failures = 0;
+ protected int failures = 0;
/* XXX Use of CocurrentHashMap is temp fix. Need to fix
* parallel accesses to DFSInputStream (through ptreads) properly */
@@ -252,24 +253,28 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
deadNodes.put(dnInfo, dnInfo);
}
- DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum
- ) throws IOException, UnresolvedLinkException {
+ DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum,
+ LocatedBlocks locatedBlocks) throws IOException, UnresolvedLinkException {
this.dfsClient = dfsClient;
this.verifyChecksum = verifyChecksum;
this.src = src;
synchronized (infoLock) {
this.cachingStrategy = dfsClient.getDefaultReadCachingStrategy();
}
- openInfo();
+ this.locatedBlocks = locatedBlocks;
+ openInfo(false);
}
/**
* Grab the open-file info from namenode
+ * @param refreshLocatedBlocks whether to re-fetch locatedblocks
*/
- void openInfo() throws IOException, UnresolvedLinkException {
+ void openInfo(boolean refreshLocatedBlocks) throws IOException,
+ UnresolvedLinkException {
final DfsClientConf conf = dfsClient.getConf();
synchronized(infoLock) {
- lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
+ lastBlockBeingWrittenLength =
+ fetchLocatedBlocksAndGetLastBlockLength(refreshLocatedBlocks);
int retriesForLastBlockLength = conf.getRetryTimesForGetLastBlockLength();
while (retriesForLastBlockLength > 0) {
// Getting last block length as -1 is a special case. When cluster
@@ -281,7 +286,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
+ "Datanodes might not have reported blocks completely."
+ " Will retry for " + retriesForLastBlockLength + " times");
waitFor(conf.getRetryIntervalForGetLastBlockLength());
- lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
+ lastBlockBeingWrittenLength =
+ fetchLocatedBlocksAndGetLastBlockLength(true);
} else {
break;
}
@@ -302,8 +308,12 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
}
- private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException {
- final LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0);
+ private long fetchLocatedBlocksAndGetLastBlockLength(boolean refresh)
+ throws IOException {
+ LocatedBlocks newInfo = locatedBlocks;
+ if (locatedBlocks == null || refresh) {
+ newInfo = dfsClient.getLocatedBlocks(src, 0);
+ }
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("newInfo = " + newInfo);
}
@@ -441,7 +451,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* @return located block
* @throws IOException
*/
- private LocatedBlock getBlockAt(long offset) throws IOException {
+ protected LocatedBlock getBlockAt(long offset) throws IOException {
synchronized(infoLock) {
assert (locatedBlocks != null) : "locatedBlocks is null";
@@ -476,7 +486,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
/** Fetch a block from namenode and cache it */
- private void fetchBlockAt(long offset) throws IOException {
+ protected void fetchBlockAt(long offset) throws IOException {
synchronized(infoLock) {
int targetBlockIdx = locatedBlocks.findBlock(offset);
if (targetBlockIdx < 0) { // block is not cached
@@ -579,7 +589,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
// Will be getting a new BlockReader.
- closeCurrentBlockReader();
+ closeCurrentBlockReaders();
//
// Connect to best DataNode for desired Block, with potential offset
@@ -620,7 +630,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
return chosenNode;
} catch (IOException ex) {
if (ex instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
- DFSClient.LOG.info("Will fetch a new encryption key and retry, "
+ DFSClient.LOG.info("Will fetch a new encryption key and retry, "
+ "encryption key was invalid when connecting to " + targetAddr
+ " : " + ex);
// The encryption key used is invalid.
@@ -631,8 +641,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
fetchBlockAt(target);
} else {
connectFailedOnce = true;
- DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block "
- +targetBlock.getBlock()+ ", add to deadNodes and continue. " + ex, ex);
+ DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
+ + ", add to deadNodes and continue. " + ex, ex);
// Put chosen node into dead list, continue
addToDeadNodes(chosenNode);
}
@@ -696,7 +706,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
"unreleased ByteBuffers allocated by read(). " +
"Please release " + builder.toString() + ".");
}
- closeCurrentBlockReader();
+ closeCurrentBlockReaders();
super.close();
}
@@ -713,12 +723,22 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* Wraps different possible read implementations so that readBuffer can be
* strategy-agnostic.
*/
- private interface ReaderStrategy {
+ interface ReaderStrategy {
public int doRead(BlockReader blockReader, int off, int len)
throws ChecksumException, IOException;
+
+ /**
+ * Copy data from the src ByteBuffer into the read buffer.
+ * @param src The src buffer where the data is copied from
+ * @param offset Useful only when the ReadStrategy is based on a byte array.
+ * Indicate the offset of the byte array for copy.
+ * @param length Useful only when the ReadStrategy is based on a byte array.
+ * Indicate the length of the data to copy.
+ */
+ public int copyFrom(ByteBuffer src, int offset, int length);
}
- private void updateReadStatistics(ReadStatistics readStatistics,
+ protected void updateReadStatistics(ReadStatistics readStatistics,
int nRead, BlockReader blockReader) {
if (nRead <= 0) return;
synchronized(infoLock) {
@@ -749,12 +769,19 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
updateReadStatistics(readStatistics, nRead, blockReader);
return nRead;
}
+
+ @Override
+ public int copyFrom(ByteBuffer src, int offset, int length) {
+ ByteBuffer writeSlice = src.duplicate();
+ writeSlice.get(buf, offset, length);
+ return length;
+ }
}
/**
* Used to read bytes into a user-supplied ByteBuffer
*/
- private class ByteBufferStrategy implements ReaderStrategy {
+ protected class ByteBufferStrategy implements ReaderStrategy {
final ByteBuffer buf;
ByteBufferStrategy(ByteBuffer buf) {
this.buf = buf;
@@ -770,6 +797,9 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
int ret = blockReader.read(buf);
success = true;
updateReadStatistics(readStatistics, ret, blockReader);
+ if (ret == 0) {
+ DFSClient.LOG.warn("zero");
+ }
return ret;
} finally {
if (!success) {
@@ -779,6 +809,15 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
}
}
+
+ @Override
+ public int copyFrom(ByteBuffer src, int offset, int length) {
+ ByteBuffer writeSlice = src.duplicate();
+ int remaining = Math.min(buf.remaining(), writeSlice.remaining());
+ writeSlice.limit(writeSlice.position() + remaining);
+ buf.put(writeSlice);
+ return remaining;
+ }
}
/* This is a used by regular read() and handles ChecksumExceptions.
@@ -837,7 +876,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
}
- private synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
+ protected synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
dfsClient.checkOpen();
if (closed.get()) {
throw new IOException("Stream closed");
@@ -926,7 +965,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
/**
* Add corrupted block replica into map.
*/
- private void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node,
+ protected void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node,
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
Set<DatanodeInfo> dnSet = null;
if((corruptedBlockMap.containsKey(blk))) {
@@ -985,8 +1024,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
} catch (InterruptedException iex) {
}
deadNodes.clear(); //2nd option is to remove only nodes[blockId]
- openInfo();
- block = getBlockAt(block.getStartOffset());
+ openInfo(true);
+ block = refreshLocatedBlock(block);
failures++;
}
}
@@ -998,7 +1037,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* @param ignoredNodes Do not choose nodes in this array (may be null)
* @return The DNAddrPair of the best node. Null if no node can be chosen.
*/
- private DNAddrPair getBestNodeDNAddrPair(LocatedBlock block,
+ protected DNAddrPair getBestNodeDNAddrPair(LocatedBlock block,
Collection<DatanodeInfo> ignoredNodes) {
DatanodeInfo[] nodes = block.getLocations();
StorageType[] storageTypes = block.getStorageTypes();
@@ -1058,15 +1097,15 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
return errMsgr.toString();
}
- private void fetchBlockByteRange(long blockStartOffset, long start, long end,
+ protected void fetchBlockByteRange(LocatedBlock block, long start, long end,
byte[] buf, int offset,
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
throws IOException {
- LocatedBlock block = getBlockAt(blockStartOffset);
+ block = refreshLocatedBlock(block);
while (true) {
DNAddrPair addressPair = chooseDataNode(block, null);
try {
- actualGetFromOneDataNode(addressPair, blockStartOffset, start, end,
+ actualGetFromOneDataNode(addressPair, block, start, end,
buf, offset, corruptedBlockMap);
return;
} catch (IOException e) {
@@ -1077,7 +1116,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
private Callable<ByteBuffer> getFromOneDataNode(final DNAddrPair datanode,
- final long blockStartOffset, final long start, final long end,
+ final LocatedBlock block, final long start, final long end,
final ByteBuffer bb,
final Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap,
final int hedgedReadId) {
@@ -1090,7 +1129,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
TraceScope scope =
Trace.startSpan("hedgedRead" + hedgedReadId, parentSpan);
try {
- actualGetFromOneDataNode(datanode, blockStartOffset, start, end, buf,
+ actualGetFromOneDataNode(datanode, block, start, end, buf,
offset, corruptedBlockMap);
return bb;
} finally {
@@ -1100,31 +1139,60 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
};
}
+ /**
+ * Used when reading contiguous blocks
+ */
private void actualGetFromOneDataNode(final DNAddrPair datanode,
- long blockStartOffset, final long start, final long end, byte[] buf,
+ LocatedBlock block, final long start, final long end, byte[] buf,
int offset, Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
throws IOException {
+ final int length = (int) (end - start + 1);
+ actualGetFromOneDataNode(datanode, block, start, end, buf,
+ new int[]{offset}, new int[]{length}, corruptedBlockMap);
+ }
+
+ /**
+ * Read data from one DataNode.
+ * @param datanode the datanode from which to read data
+ * @param block the located block containing the requested data
+ * @param startInBlk the startInBlk offset of the block
+ * @param endInBlk the endInBlk offset of the block
+ * @param buf the given byte array into which the data is read
+ * @param offsets the data may be read into multiple segments of the buf
+ * (when reading a striped block). this array indicates the
+ * offset of each buf segment.
+ * @param lengths the length of each buf segment
+ * @param corruptedBlockMap map recording list of datanodes with corrupted
+ * block replica
+ */
+ void actualGetFromOneDataNode(final DNAddrPair datanode,
+ LocatedBlock block, final long startInBlk, final long endInBlk,
+ byte[] buf, int[] offsets, int[] lengths,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+ throws IOException {
DFSClientFaultInjector.get().startFetchFromDatanode();
int refetchToken = 1; // only need to get a new access token once
int refetchEncryptionKey = 1; // only need to get a new encryption key once
+ final int len = (int) (endInBlk - startInBlk + 1);
+ checkReadPortions(offsets, lengths, len);
while (true) {
// cached block locations may have been updated by chooseDataNode()
// or fetchBlockAt(). Always get the latest list of locations at the
// start of the loop.
- LocatedBlock block = getBlockAt(blockStartOffset);
+ block = refreshLocatedBlock(block);
BlockReader reader = null;
try {
DFSClientFaultInjector.get().fetchFromDatanodeException();
- int len = (int) (end - start + 1);
- reader = getBlockReader(block, start, len, datanode.addr,
+ reader = getBlockReader(block, startInBlk, len, datanode.addr,
datanode.storageType, datanode.info);
- int nread = reader.readAll(buf, offset, len);
- updateReadStatistics(readStatistics, nread, reader);
-
- if (nread != len) {
- throw new IOException("truncated return from reader.read(): " +
- "excpected " + len + ", got " + nread);
+ for (int i = 0; i < offsets.length; i++) {
+ int nread = reader.readAll(buf, offsets[i], lengths[i]);
+ updateReadStatistics(readStatistics, nread, reader);
+ if (nread != lengths[i]) {
+ throw new IOException("truncated return from reader.read(): " +
+ "excpected " + lengths[i] + ", got " + nread);
+ }
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
return;
@@ -1169,11 +1237,40 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
/**
- * Like {@link #fetchBlockByteRange} except we start up a second, parallel,
+ * Refresh cached block locations.
+ * @param block The currently cached block locations
+ * @return Refreshed block locations
+ * @throws IOException
+ */
+ protected LocatedBlock refreshLocatedBlock(LocatedBlock block)
+ throws IOException {
+ return getBlockAt(block.getStartOffset());
+ }
+
+ /**
+ * This method verifies that the read portions are valid and do not overlap
+ * with each other.
+ */
+ private void checkReadPortions(int[] offsets, int[] lengths, int totalLen) {
+ Preconditions.checkArgument(offsets.length == lengths.length && offsets.length > 0);
+ int sum = 0;
+ for (int i = 0; i < lengths.length; i++) {
+ if (i > 0) {
+ int gap = offsets[i] - offsets[i - 1];
+ // make sure read portions do not overlap with each other
+ Preconditions.checkArgument(gap >= lengths[i - 1]);
+ }
+ sum += lengths[i];
+ }
+ Preconditions.checkArgument(sum == totalLen);
+ }
+
+ /**
+ * Like {@link #fetchBlockByteRange}except we start up a second, parallel,
* 'hedged' read if the first read is taking longer than configured amount of
* time. We then wait on which ever read returns first.
*/
- private void hedgedFetchBlockByteRange(long blockStartOffset, long start,
+ private void hedgedFetchBlockByteRange(LocatedBlock block, long start,
long end, byte[] buf, int offset,
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
throws IOException {
@@ -1186,7 +1283,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
ByteBuffer bb = null;
int len = (int) (end - start + 1);
int hedgedReadId = 0;
- LocatedBlock block = getBlockAt(blockStartOffset);
+ block = refreshLocatedBlock(block);
while (true) {
// see HDFS-6591, this metric is used to verify/catch unnecessary loops
hedgedReadOpsLoopNumForTesting++;
@@ -1198,7 +1295,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
chosenNode = chooseDataNode(block, ignored);
bb = ByteBuffer.wrap(buf, offset, len);
Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
- chosenNode, block.getStartOffset(), start, end, bb,
+ chosenNode, block, start, end, bb,
corruptedBlockMap, hedgedReadId++);
Future<ByteBuffer> firstRequest = hedgedService
.submit(getFromDataNodeCallable);
@@ -1235,7 +1332,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
bb = ByteBuffer.allocate(len);
Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
- chosenNode, block.getStartOffset(), start, end, bb,
+ chosenNode, block, start, end, bb,
corruptedBlockMap, hedgedReadId++);
Future<ByteBuffer> oneMoreRequest = hedgedService
.submit(getFromDataNodeCallable);
@@ -1319,7 +1416,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* @return true if block access token has expired or invalid and it should be
* refetched
*/
- private static boolean tokenRefetchNeeded(IOException ex,
+ protected static boolean tokenRefetchNeeded(IOException ex,
InetSocketAddress targetAddr) {
/*
* Get a new access token and retry. Retry is needed in 2 cases. 1)
@@ -1389,13 +1486,11 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
try {
if (dfsClient.isHedgedReadsEnabled()) {
- hedgedFetchBlockByteRange(blk.getStartOffset(), targetStart,
- targetStart + bytesToRead - 1, buffer, offset,
- corruptedBlockMap);
+ hedgedFetchBlockByteRange(blk, targetStart,
+ targetStart + bytesToRead - 1, buffer, offset, corruptedBlockMap);
} else {
- fetchBlockByteRange(blk.getStartOffset(), targetStart,
- targetStart + bytesToRead - 1, buffer, offset,
- corruptedBlockMap);
+ fetchBlockByteRange(blk, targetStart, targetStart + bytesToRead - 1,
+ buffer, offset, corruptedBlockMap);
}
} finally {
// Check and report if any block replicas are corrupted.
@@ -1427,7 +1522,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* @param corruptedBlockMap map of corrupted blocks
* @param dataNodeCount number of data nodes who contains the block replicas
*/
- private void reportCheckSumFailure(
+ protected void reportCheckSumFailure(
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap,
int dataNodeCount) {
if (corruptedBlockMap.isEmpty()) {
@@ -1556,7 +1651,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
/**
*/
@Override
- public synchronized long getPos() throws IOException {
+ public synchronized long getPos() {
return pos;
}
@@ -1590,7 +1685,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
/** Utility class to encapsulate data node info and its address. */
- private static final class DNAddrPair {
+ static final class DNAddrPair {
final DatanodeInfo info;
final InetSocketAddress addr;
final StorageType storageType;
@@ -1627,7 +1722,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
}
- private void closeCurrentBlockReader() {
+ protected void closeCurrentBlockReaders() {
if (blockReader == null) return;
// Close the current block reader so that the new caching settings can
// take effect immediately.
@@ -1647,7 +1742,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
this.cachingStrategy =
new CachingStrategy.Builder(this.cachingStrategy).setReadahead(readahead).build();
}
- closeCurrentBlockReader();
+ closeCurrentBlockReaders();
}
@Override
@@ -1657,7 +1752,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
this.cachingStrategy =
new CachingStrategy.Builder(this.cachingStrategy).setDropBehind(dropBehind).build();
}
- closeCurrentBlockReader();
+ closeCurrentBlockReaders();
}
/**
@@ -1815,6 +1910,6 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
@Override
public synchronized void unbuffer() {
- closeCurrentBlockReader();
+ closeCurrentBlockReaders();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 68cc155..43e0eb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -343,7 +343,7 @@ public class TestDFSClientRetries {
// we're starting a new operation on the user level.
doAnswer(new FailNTimesAnswer(preSpyNN, maxBlockAcquires))
.when(spyNN).getBlockLocations(anyString(), anyLong(), anyLong());
- is.openInfo();
+ is.openInfo(true);
// Seek to beginning forces a reopen of the BlockReader - otherwise it'll
// just keep reading on the existing stream and the fact that we've poisoned
// the block info won't do anything.
[33/48] hadoop git commit: MAPREDUCE-6376. Add avro binary support
for jhist files. Contributed by Ray Chiang
Posted by zj...@apache.org.
MAPREDUCE-6376. Add avro binary support for jhist files. Contributed by Ray Chiang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23ac91c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23ac91c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23ac91c9
Branch: refs/heads/YARN-2928
Commit: 23ac91c90964e0af9d1f6b9a4b6de871b9be9e72
Parents: 14bfd5d
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 16:00:03 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../jobhistory/JobHistoryEventHandler.java | 19 +++++++++++--
.../hadoop/mapreduce/jobhistory/TestEvents.java | 3 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 +++++
.../mapreduce/jobhistory/EventReader.java | 12 ++++----
.../mapreduce/jobhistory/EventWriter.java | 29 ++++++++++++++++----
.../src/main/resources/mapred-default.xml | 9 ++++++
7 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d170b55..49a873d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -396,6 +396,9 @@ Release 2.8.0 - UNRELEASED
OPTIMIZATIONS
+ MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via
+ jlowe)
+
BUG FIXES
MAPREDUCE-6314. TestPipeApplication fails on trunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 5208874..a8f10d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -108,7 +108,8 @@ public class JobHistoryEventHandler extends AbstractService
private int numUnflushedCompletionEvents = 0;
private boolean isTimerActive;
-
+ private EventWriter.WriteMode jhistMode =
+ EventWriter.WriteMode.JSON;
protected BlockingQueue<JobHistoryEvent> eventQueue =
new LinkedBlockingQueue<JobHistoryEvent>();
@@ -282,6 +283,20 @@ public class JobHistoryEventHandler extends AbstractService
LOG.info("Emitting job history data to the timeline server is not enabled");
}
+ // Flag for setting
+ String jhistFormat = conf.get(JHAdminConfig.MR_HS_JHIST_FORMAT,
+ JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT);
+ if (jhistFormat.equals("json")) {
+ jhistMode = EventWriter.WriteMode.JSON;
+ } else if (jhistFormat.equals("binary")) {
+ jhistMode = EventWriter.WriteMode.BINARY;
+ } else {
+ LOG.warn("Unrecognized value '" + jhistFormat + "' for property " +
+ JHAdminConfig.MR_HS_JHIST_FORMAT + ". Valid values are " +
+ "'json' or 'binary'. Falling back to default value '" +
+ JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT + "'.");
+ }
+
super.serviceInit(conf);
}
@@ -457,7 +472,7 @@ public class JobHistoryEventHandler extends AbstractService
protected EventWriter createEventWriter(Path historyFilePath)
throws IOException {
FSDataOutputStream out = stagingDirFS.create(historyFilePath, true);
- return new EventWriter(out);
+ return new EventWriter(out, this.jhistMode);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 27fcb33..e7d5006 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -191,7 +191,8 @@ public class TestEvents {
ByteArrayOutputStream output = new ByteArrayOutputStream();
FSDataOutputStream fsOutput = new FSDataOutputStream(output,
new FileSystem.Statistics("scheme"));
- EventWriter writer = new EventWriter(fsOutput);
+ EventWriter writer = new EventWriter(fsOutput,
+ EventWriter.WriteMode.JSON);
writer.write(getJobPriorityChangedEvent());
writer.write(getJobStatusChangedEvent());
writer.write(getTaskUpdatedEvent());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index a97c2ca..86dfad3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -221,4 +221,11 @@ public class JHAdminConfig {
+ "jobname.limit";
public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50;
+ /**
+ * Settings for .jhist file format.
+ */
+ public static final String MR_HS_JHIST_FORMAT =
+ MR_HISTORY_PREFIX + "jhist.format";
+ public static final String DEFAULT_MR_HS_JHIST_FORMAT =
+ "json";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
index e08a929..9898c2d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
@@ -66,16 +66,18 @@ public class EventReader implements Closeable {
public EventReader(DataInputStream in) throws IOException {
this.in = in;
this.version = in.readLine();
-
- if (!EventWriter.VERSION.equals(version)) {
- throw new IOException("Incompatible event log version: "+version);
- }
Schema myschema = new SpecificData(Event.class.getClassLoader()).getSchema(Event.class);
Schema.Parser parser = new Schema.Parser();
this.schema = parser.parse(in.readLine());
this.reader = new SpecificDatumReader(schema, myschema);
- this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
+ if (EventWriter.VERSION.equals(version)) {
+ this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
+ } else if (EventWriter.VERSION_BINARY.equals(version)) {
+ this.decoder = DecoderFactory.get().binaryDecoder(in, null);
+ } else {
+ throw new IOException("Incompatible event log version: " + version);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
index a548dfe..29489a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
@@ -43,20 +43,37 @@ import org.apache.hadoop.mapreduce.Counters;
*/
class EventWriter {
static final String VERSION = "Avro-Json";
+ static final String VERSION_BINARY = "Avro-Binary";
private FSDataOutputStream out;
private DatumWriter<Event> writer =
new SpecificDatumWriter<Event>(Event.class);
private Encoder encoder;
private static final Log LOG = LogFactory.getLog(EventWriter.class);
-
- EventWriter(FSDataOutputStream out) throws IOException {
+ public enum WriteMode { JSON, BINARY }
+ private final WriteMode writeMode;
+ private final boolean jsonOutput; // Cache value while we have 2 modes
+
+ EventWriter(FSDataOutputStream out, WriteMode mode) throws IOException {
this.out = out;
- out.writeBytes(VERSION);
+ this.writeMode = mode;
+ if (this.writeMode==WriteMode.JSON) {
+ this.jsonOutput = true;
+ out.writeBytes(VERSION);
+ } else if (this.writeMode==WriteMode.BINARY) {
+ this.jsonOutput = false;
+ out.writeBytes(VERSION_BINARY);
+ } else {
+ throw new IOException("Unknown mode: " + mode);
+ }
out.writeBytes("\n");
out.writeBytes(Event.SCHEMA$.toString());
out.writeBytes("\n");
- this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
+ if (!this.jsonOutput) {
+ this.encoder = EncoderFactory.get().binaryEncoder(out, null);
+ } else {
+ this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
+ }
}
synchronized void write(HistoryEvent event) throws IOException {
@@ -65,7 +82,9 @@ class EventWriter {
wrapper.setEvent(event.getDatum());
writer.write(wrapper, encoder);
encoder.flush();
- out.writeBytes("\n");
+ if (this.jsonOutput) {
+ out.writeBytes("\n");
+ }
}
void flush() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index c89a566..8d3450b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1721,6 +1721,15 @@
</property>
<property>
+ <description>
+ File format the AM will use when generating the .jhist file. Valid
+ values are "json" for text output and "binary" for faster parsing.
+ </description>
+ <name>mapreduce.jobhistory.jhist.format</name>
+ <value>json</value>
+</property>
+
+<property>
<name>mapreduce.job.heap.memory-mb.ratio</name>
<value>0.8</value>
<description>The ratio of heap-size to container-size. If no -Xmx is
[11/48] hadoop git commit: Revert "HADOOP-12009 Clarify
FileSystem.listStatus() sorting order & fix
FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)"
Posted by zj...@apache.org.
Revert "HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)"
This reverts commit 3dfa8161f9412bcb040f3c29c471344f25f24337.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/405b0982
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/405b0982
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/405b0982
Branch: refs/heads/YARN-2928
Commit: 405b0982b600a5c7c44ebf8c6644db15a43e2091
Parents: d963454
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 30 11:54:12 2015 +0100
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ----
.../main/java/org/apache/hadoop/fs/FileSystem.java | 17 +----------------
.../src/site/markdown/filesystem/filesystem.md | 4 ----
.../hadoop/fs/FileSystemContractBaseTest.java | 11 +++--------
4 files changed, 4 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2cf9509..c010ff1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -664,10 +664,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
occurs (Jason Lowe via jeagles)
-
- HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix
- FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)
-
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c98074a..c73caf7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1498,9 +1498,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
+ *
* @param f given path
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
@@ -1542,9 +1540,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given path using the user-supplied path
* filter.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
*
* @param f
* a path name
@@ -1565,9 +1560,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given list of paths using default
* path filter.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
*
* @param files
* a list of paths
@@ -1584,9 +1576,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given list of paths using user-supplied
* path filter.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
*
* @param files
* a list of paths
@@ -1747,8 +1736,6 @@ public abstract class FileSystem extends Configured implements Closeable {
* while consuming the entries. Each file system implementation should
* override this method and provide a more efficient implementation, if
* possible.
- * Does not guarantee to return the iterator that traverses statuses
- * of the files in a sorted order.
*
* @param p target path
* @return remote iterator
@@ -1776,8 +1763,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* List the statuses and block locations of the files in the given path.
- * Does not guarantee to return the iterator that traverses statuses
- * of the files in a sorted order.
*
* If the path is a directory,
* if recursive is false, returns files in the directory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index f323374..84e3755 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -183,10 +183,6 @@ to the same path:
forall fs in listStatus(Path) :
fs == getFileStatus(fs.path)
-**Ordering of results**: there is no guarantee of ordering of the listed entries.
-While HDFS currently returns an alphanumerically sorted list, neither the Posix `readdir()`
-nor Java's `File.listFiles()` API calls define any ordering of returned values. Applications
-which require a uniform sort order on the results must perform the sorting themselves.
### Atomicity and Consistency
http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 495af00..2ca81e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import junit.framework.TestCase;
@@ -225,13 +224,9 @@ public abstract class FileSystemContractBaseTest extends TestCase {
paths = fs.listStatus(path("/test/hadoop"));
assertEquals(3, paths.length);
- ArrayList<String> list = new ArrayList<String>();
- for (FileStatus fileState : paths) {
- list.add(fileState.getPath().toString());
- }
- assertTrue(list.contains(path("/test/hadoop/a")));
- assertTrue(list.contains(path("/test/hadoop/b")));
- assertTrue(list.contains(path("/test/hadoop/c")));
+ assertEquals(path("/test/hadoop/a"), paths[0].getPath());
+ assertEquals(path("/test/hadoop/b"), paths[1].getPath());
+ assertEquals(path("/test/hadoop/c"), paths[2].getPath());
paths = fs.listStatus(path("/test/hadoop/a"));
assertEquals(0, paths.length);
[36/48] hadoop git commit: YARN-3508. Prevent processing preemption
events on the main RM dispatcher. (Varun Saxena via wangda)
Posted by zj...@apache.org.
YARN-3508. Prevent processing preemption events on the main RM dispatcher. (Varun Saxena via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5192ca9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5192ca9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5192ca9
Branch: refs/heads/YARN-2928
Commit: d5192ca993d8c76fe0b51c3a91967fb0c2bbf0a1
Parents: 2820f1f
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Jul 1 17:32:22 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/ResourceManager.java | 34 ---------
.../monitor/SchedulingEditPolicy.java | 6 +-
.../monitor/SchedulingMonitor.java | 3 +-
.../ProportionalCapacityPreemptionPolicy.java | 42 ++++++-----
.../scheduler/ContainerPreemptEvent.java | 8 +-
.../scheduler/ContainerPreemptEventType.java | 26 -------
.../scheduler/capacity/CapacityScheduler.java | 24 ++++++
.../scheduler/event/SchedulerEventType.java | 7 +-
.../resourcemanager/TestRMDispatcher.java | 79 ++++++++++++++++++++
...estProportionalCapacityPreemptionPolicy.java | 19 +++--
...pacityPreemptionPolicyForNodePartitions.java | 12 ++-
12 files changed, 161 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c4c7fa9..3e42e61 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -704,6 +704,9 @@ Release 2.7.2 - UNRELEASED
YARN-3793. Several NPEs when deleting local files on NM recovery (Varun
Saxena via jlowe)
+ YARN-3508. Prevent processing preemption events on the main RM dispatcher.
+ (Varun Saxena via wangda)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 3302320..b235058 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -76,7 +76,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
@@ -624,9 +623,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
SchedulingEditPolicy.class);
if (policies.size() > 0) {
- rmDispatcher.register(ContainerPreemptEventType.class,
- new RMContainerPreemptEventDispatcher(
- (PreemptableResourceScheduler) scheduler));
for (SchedulingEditPolicy policy : policies) {
LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
// periodically check whether we need to take action to guarantee
@@ -797,36 +793,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
@Private
- public static final class
- RMContainerPreemptEventDispatcher
- implements EventHandler<ContainerPreemptEvent> {
-
- private final PreemptableResourceScheduler scheduler;
-
- public RMContainerPreemptEventDispatcher(
- PreemptableResourceScheduler scheduler) {
- this.scheduler = scheduler;
- }
-
- @Override
- public void handle(ContainerPreemptEvent event) {
- ApplicationAttemptId aid = event.getAppId();
- RMContainer container = event.getContainer();
- switch (event.getType()) {
- case DROP_RESERVATION:
- scheduler.dropContainerReservation(container);
- break;
- case PREEMPT_CONTAINER:
- scheduler.preemptContainer(aid, container);
- break;
- case KILL_CONTAINER:
- scheduler.killContainer(container);
- break;
- }
- }
- }
-
- @Private
public static final class ApplicationAttemptEventDispatcher implements
EventHandler<RMAppAttemptEvent> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
index 1ebc19f..0d587d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
@@ -18,14 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
public interface SchedulingEditPolicy {
- public void init(Configuration config,
- EventHandler<ContainerPreemptEvent> dispatcher,
+ public void init(Configuration config, RMContext context,
PreemptableResourceScheduler scheduler);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
index 1682f7d..d4c129b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
@@ -54,9 +54,8 @@ public class SchedulingMonitor extends AbstractService {
return scheduleEditPolicy;
}
- @SuppressWarnings("unchecked")
public void serviceInit(Configuration conf) throws Exception {
- scheduleEditPolicy.init(conf, rmContext.getDispatcher().getEventHandler(),
+ scheduleEditPolicy.init(conf, rmContext,
(PreemptableResourceScheduler) rmContext.getScheduler());
this.monitorInterval = scheduleEditPolicy.getMonitoringInterval();
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1f47b5f..5a20a6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -38,13 +38,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -52,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -118,8 +118,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
public static final String NATURAL_TERMINATION_FACTOR =
"yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor";
- // the dispatcher to send preempt and kill events
- public EventHandler<ContainerPreemptEvent> dispatcher;
+ private RMContext rmContext;
private final Clock clock;
private double maxIgnoredOverCapacity;
@@ -141,20 +140,17 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
}
public ProportionalCapacityPreemptionPolicy(Configuration config,
- EventHandler<ContainerPreemptEvent> dispatcher,
- CapacityScheduler scheduler) {
- this(config, dispatcher, scheduler, new SystemClock());
+ RMContext context, CapacityScheduler scheduler) {
+ this(config, context, scheduler, new SystemClock());
}
public ProportionalCapacityPreemptionPolicy(Configuration config,
- EventHandler<ContainerPreemptEvent> dispatcher,
- CapacityScheduler scheduler, Clock clock) {
- init(config, dispatcher, scheduler);
+ RMContext context, CapacityScheduler scheduler, Clock clock) {
+ init(config, context, scheduler);
this.clock = clock;
}
- public void init(Configuration config,
- EventHandler<ContainerPreemptEvent> disp,
+ public void init(Configuration config, RMContext context,
PreemptableResourceScheduler sched) {
LOG.info("Preemption monitor:" + this.getClass().getCanonicalName());
assert null == scheduler : "Unexpected duplicate call to init";
@@ -163,7 +159,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
sched.getClass().getCanonicalName() + " not instance of " +
CapacityScheduler.class.getCanonicalName());
}
- dispatcher = disp;
+ rmContext = context;
scheduler = (CapacityScheduler) sched;
maxIgnoredOverCapacity = config.getDouble(MAX_IGNORED_OVER_CAPACITY, 0.1);
naturalTerminationFactor =
@@ -196,6 +192,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
* @param root the root of the CapacityScheduler queue hierarchy
* @param clusterResources the total amount of resources in the cluster
*/
+ @SuppressWarnings("unchecked")
private void containerBasedPreemptOrKill(CSQueue root,
Resource clusterResources) {
// All partitions to look at
@@ -248,8 +245,9 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
// preempt (or kill) the selected containers
for (Map.Entry<ApplicationAttemptId,Set<RMContainer>> e
: toPreempt.entrySet()) {
+ ApplicationAttemptId appAttemptId = e.getKey();
if (LOG.isDebugEnabled()) {
- LOG.debug("Send to scheduler: in app=" + e.getKey()
+ LOG.debug("Send to scheduler: in app=" + appAttemptId
+ " #containers-to-be-preempted=" + e.getValue().size());
}
for (RMContainer container : e.getValue()) {
@@ -257,13 +255,15 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
if (preempted.get(container) != null &&
preempted.get(container) + maxWaitTime < clock.getTime()) {
// kill it
- dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
- ContainerPreemptEventType.KILL_CONTAINER));
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(appAttemptId, container,
+ SchedulerEventType.KILL_CONTAINER));
preempted.remove(container);
} else {
//otherwise just send preemption events
- dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
- ContainerPreemptEventType.PREEMPT_CONTAINER));
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(appAttemptId, container,
+ SchedulerEventType.PREEMPT_CONTAINER));
if (preempted.get(container) == null) {
preempted.put(container, clock.getTime());
}
@@ -735,6 +735,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
* Given a target preemption for a specific application, select containers
* to preempt (after unreserving all reservation for that app).
*/
+ @SuppressWarnings("unchecked")
private void preemptFrom(FiCaSchedulerApp app,
Resource clusterResource, Map<String, Resource> resToObtainByPartition,
List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
@@ -758,8 +759,9 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
clusterResource, preemptMap);
if (!observeOnly) {
- dispatcher.handle(new ContainerPreemptEvent(appId, c,
- ContainerPreemptEventType.DROP_RESERVATION));
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(
+ appId, c, SchedulerEventType.DROP_RESERVATION));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
index 8eba48d..7ab2758 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
@@ -19,20 +19,20 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
/**
* Simple event class used to communicate containers unreservations, preemption, killing
*/
-public class ContainerPreemptEvent
- extends AbstractEvent<ContainerPreemptEventType> {
+public class ContainerPreemptEvent extends SchedulerEvent {
private final ApplicationAttemptId aid;
private final RMContainer container;
public ContainerPreemptEvent(ApplicationAttemptId aid, RMContainer container,
- ContainerPreemptEventType type) {
+ SchedulerEventType type) {
super(type);
this.aid = aid;
this.container = container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java
deleted file mode 100644
index a70a836..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-
-public enum ContainerPreemptEventType {
-
- DROP_RESERVATION,
- PREEMPT_CONTAINER,
- KILL_CONTAINER
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index f1d0f9c..141aa7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -1346,6 +1347,29 @@ public class CapacityScheduler extends
RMContainerEventType.EXPIRE);
}
break;
+ case DROP_RESERVATION:
+ {
+ ContainerPreemptEvent dropReservationEvent = (ContainerPreemptEvent)event;
+ RMContainer container = dropReservationEvent.getContainer();
+ dropContainerReservation(container);
+ }
+ break;
+ case PREEMPT_CONTAINER:
+ {
+ ContainerPreemptEvent preemptContainerEvent =
+ (ContainerPreemptEvent)event;
+ ApplicationAttemptId aid = preemptContainerEvent.getAppId();
+ RMContainer containerToBePreempted = preemptContainerEvent.getContainer();
+ preemptContainer(aid, containerToBePreempted);
+ }
+ break;
+ case KILL_CONTAINER:
+ {
+ ContainerPreemptEvent killContainerEvent = (ContainerPreemptEvent)event;
+ RMContainer containerToBeKilled = killContainerEvent.getContainer();
+ killContainer(containerToBeKilled);
+ }
+ break;
default:
LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 13aecb3..9de935b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -36,5 +36,10 @@ public enum SchedulerEventType {
APP_ATTEMPT_REMOVED,
// Source: ContainerAllocationExpirer
- CONTAINER_EXPIRED
+ CONTAINER_EXPIRED,
+
+ // Source: SchedulingEditPolicy
+ DROP_RESERVATION,
+ PREEMPT_CONTAINER,
+ KILL_CONTAINER
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java
new file mode 100644
index 0000000..db7c96a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.SchedulerEventDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestRMDispatcher {
+
+ @SuppressWarnings("unchecked")
+ @Test(timeout=10000)
+ public void testSchedulerEventDispatcherForPreemptionEvents() {
+ AsyncDispatcher rmDispatcher = new AsyncDispatcher();
+ CapacityScheduler sched = spy(new CapacityScheduler());
+ YarnConfiguration conf = new YarnConfiguration();
+ SchedulerEventDispatcher schedulerDispatcher =
+ new SchedulerEventDispatcher(sched);
+ rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
+ rmDispatcher.init(conf);
+ rmDispatcher.start();
+ schedulerDispatcher.init(conf);
+ schedulerDispatcher.start();
+ try {
+ ApplicationAttemptId appAttemptId = mock(ApplicationAttemptId.class);
+ RMContainer container = mock(RMContainer.class);
+ ContainerPreemptEvent event1 = new ContainerPreemptEvent(
+ appAttemptId, container, SchedulerEventType.DROP_RESERVATION);
+ rmDispatcher.getEventHandler().handle(event1);
+ ContainerPreemptEvent event2 = new ContainerPreemptEvent(
+ appAttemptId, container, SchedulerEventType.KILL_CONTAINER);
+ rmDispatcher.getEventHandler().handle(event2);
+ ContainerPreemptEvent event3 = new ContainerPreemptEvent(
+ appAttemptId, container, SchedulerEventType.PREEMPT_CONTAINER);
+ rmDispatcher.getEventHandler().handle(event3);
+ // Wait for events to be processed by scheduler dispatcher.
+ Thread.sleep(1000);
+ verify(sched, times(3)).handle(any(SchedulerEvent.class));
+ verify(sched).dropContainerReservation(container);
+ verify(sched).preemptContainer(appAttemptId, container);
+ verify(sched).killContainer(container);
+ } catch (InterruptedException e) {
+ Assert.fail();
+ } finally {
+ schedulerDispatcher.stop();
+ rmDispatcher.stop();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 6c0ed6c..2c0c6d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -23,8 +23,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.Pro
import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.OBSERVE_ONLY;
import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND;
import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType.KILL_CONTAINER;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType.PREEMPT_CONTAINER;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.KILL_CONTAINER;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.PREEMPT_CONTAINER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -66,7 +67,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.resource.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -104,7 +106,7 @@ public class TestProportionalCapacityPreemptionPolicy {
RMContext rmContext = null;
RMNodeLabelsManager lm = null;
CapacitySchedulerConfiguration schedConf = null;
- EventHandler<ContainerPreemptEvent> mDisp = null;
+ EventHandler<SchedulerEvent> mDisp = null;
ResourceCalculator rc = new DefaultResourceCalculator();
Resource clusterResources = null;
final ApplicationAttemptId appA = ApplicationAttemptId.newInstance(
@@ -164,6 +166,9 @@ public class TestProportionalCapacityPreemptionPolicy {
when(mCS.getRMContext()).thenReturn(rmContext);
when(rmContext.getNodeLabelManager()).thenReturn(lm);
mDisp = mock(EventHandler.class);
+ Dispatcher disp = mock(Dispatcher.class);
+ when(rmContext.getDispatcher()).thenReturn(disp);
+ when(disp.getEventHandler()).thenReturn(mDisp);
rand = new Random();
long seed = rand.nextLong();
System.out.println(name.getMethodName() + " SEED: " + seed);
@@ -866,12 +871,12 @@ public class TestProportionalCapacityPreemptionPolicy {
static class IsPreemptionRequestFor
extends ArgumentMatcher<ContainerPreemptEvent> {
private final ApplicationAttemptId appAttId;
- private final ContainerPreemptEventType type;
+ private final SchedulerEventType type;
IsPreemptionRequestFor(ApplicationAttemptId appAttId) {
this(appAttId, PREEMPT_CONTAINER);
}
IsPreemptionRequestFor(ApplicationAttemptId appAttId,
- ContainerPreemptEventType type) {
+ SchedulerEventType type) {
this.appAttId = appAttId;
this.type = type;
}
@@ -888,7 +893,7 @@ public class TestProportionalCapacityPreemptionPolicy {
ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) {
ProportionalCapacityPreemptionPolicy policy =
- new ProportionalCapacityPreemptionPolicy(conf, mDisp, mCS, mClock);
+ new ProportionalCapacityPreemptionPolicy(conf, rmContext, mCS, mClock);
ParentQueue mRoot = buildMockRootQueue(rand, qData);
when(mCS.getRootQueue()).thenReturn(mRoot);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index e13320c..114769c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -50,13 +50,13 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -92,7 +93,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
private Configuration conf = null;
private CapacitySchedulerConfiguration csConf = null;
private CapacityScheduler cs = null;
- private EventHandler<ContainerPreemptEvent> mDisp = null;
+ private EventHandler<SchedulerEvent> mDisp = null;
private ProportionalCapacityPreemptionPolicy policy = null;
private Resource clusterResource = null;
@@ -125,11 +126,14 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
rmContext = mock(RMContext.class);
when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+ Dispatcher disp = mock(Dispatcher.class);
+ when(rmContext.getDispatcher()).thenReturn(disp);
+ when(disp.getEventHandler()).thenReturn(mDisp);
csConf = new CapacitySchedulerConfiguration();
when(cs.getConfiguration()).thenReturn(csConf);
when(cs.getRMContext()).thenReturn(rmContext);
- policy = new ProportionalCapacityPreemptionPolicy(conf, mDisp, cs, mClock);
+ policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, mClock);
partitionToResource = new HashMap<>();
nodeIdToSchedulerNodes = new HashMap<>();
nameToCSQueues = new HashMap<>();
@@ -828,7 +832,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
when(cs.getClusterResource()).thenReturn(clusterResource);
mockApplications(appsConfig);
- policy = new ProportionalCapacityPreemptionPolicy(conf, mDisp, cs, mClock);
+ policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, mClock);
}
private void mockContainers(String containersConfig, ApplicationAttemptId attemptId,
[02/48] hadoop git commit: HADOOP-12089. StorageException complaining
" no lease ID" when updating FolderLastModifiedTime in WASB. Contributed by
Duo Xu.
Posted by zj...@apache.org.
HADOOP-12089. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. Contributed by Duo Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3aff2fdb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3aff2fdb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3aff2fdb
Branch: refs/heads/YARN-2928
Commit: 3aff2fdbe38fcb49e23908a1bfa83743fa604e70
Parents: 6f780d1
Author: cnauroth <cn...@apache.org>
Authored: Mon Jun 29 13:48:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 5 ++---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aff2fdb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9008ead..a9b44e3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -893,6 +893,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12119. hadoop fs -expunge does not work for federated namespace
(J.Andreina via vinayakumarb)
+ HADOOP-12089. StorageException complaining " no lease ID" when updating
+ FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aff2fdb/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 623645a..a567b33 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1500,7 +1500,7 @@ public class NativeAzureFileSystem extends FileSystem {
createPermissionStatus(FsPermission.getDefault()));
} else {
if (!skipParentFolderLastModifidedTimeUpdate) {
- store.updateFolderLastModifiedTime(parentKey, null);
+ updateParentFolderLastModifiedTime(key);
}
}
}
@@ -1561,9 +1561,8 @@ public class NativeAzureFileSystem extends FileSystem {
// Update parent directory last modified time
Path parent = absolutePath.getParent();
if (parent != null && parent.getParent() != null) { // not root
- String parentKey = pathToKey(parent);
if (!skipParentFolderLastModifidedTimeUpdate) {
- store.updateFolderLastModifiedTime(parentKey, null);
+ updateParentFolderLastModifiedTime(key);
}
}
instrumentation.directoryDeleted();
[20/48] hadoop git commit: HADOOP-12149. copy all of test-patch
BINDIR prior to re-exec (aw)
Posted by zj...@apache.org.
HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c68da70b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c68da70b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c68da70b
Branch: refs/heads/YARN-2928
Commit: c68da70b70cc050234d05612ca75b442efe8db06
Parents: 405b098
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jun 30 11:31:26 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 4 ++--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c68da70b/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index cd91a5c..efcd614 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1521,8 +1521,8 @@ function check_reexec
cd "${CWD}"
mkdir -p "${PATCH_DIR}/dev-support-test"
- cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test"
- cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test"
+ (cd "${BINDIR}"; tar cpf - . ) \
+ | (cd "${PATCH_DIR}/dev-support-test"; tar xpf - )
big_console_header "exec'ing test-patch.sh now..."
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c68da70b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c010ff1..e332ea8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -232,6 +232,8 @@ Trunk (Unreleased)
HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
+ HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
[15/48] hadoop git commit: HDFS-8493. Consolidate truncate() related
implementation in a single class. Contributed by Rakesh R.
Posted by zj...@apache.org.
HDFS-8493. Consolidate truncate() related implementation in a single class. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e39b962
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e39b962
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e39b962
Branch: refs/heads/YARN-2928
Commit: 4e39b9622bd871c9e16cc22fdc85801fd8915e72
Parents: c51682b
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Jun 29 16:40:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirTruncateOp.java | 361 +++++++++++++++++++
.../hdfs/server/namenode/FSDirectory.java | 95 -----
.../hdfs/server/namenode/FSEditLogLoader.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 237 ++----------
.../hdfs/server/namenode/TestFileTruncate.java | 8 +-
6 files changed, 402 insertions(+), 308 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 108a6c0..3535f90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and
DatanodeStorageInfo. (Zhe Zhang via wang)
+ HDFS-8493. Consolidate truncate() related implementation in a single class.
+ (Rakesh R via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
new file mode 100644
index 0000000..9fc9def
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -0,0 +1,361 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionContiguous;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.RecoverLeaseOp;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Helper class to perform truncate operation.
+ */
+final class FSDirTruncateOp {
+
+ /**
+ * Private constructor for preventing FSDirTruncateOp object creation.
+ * Static-only class.
+ */
+ private FSDirTruncateOp() {}
+
+ /**
+ * Truncate a file to a given size.
+ *
+ * @param fsn namespace
+ * @param srcArg path name
+ * @param newLength the target file size
+ * @param clientName client name
+ * @param clientMachine client machine info
+ * @param mtime modified time
+ * @param toRemoveBlocks to be removed blocks
+ * @param pc permission checker to check fs permission
+ * @return tuncate result
+ * @throws IOException
+ */
+ static TruncateResult truncate(final FSNamesystem fsn, final String srcArg,
+ final long newLength, final String clientName,
+ final String clientMachine, final long mtime,
+ final BlocksMapUpdateInfo toRemoveBlocks, final FSPermissionChecker pc)
+ throws IOException, UnresolvedLinkException {
+ assert fsn.hasWriteLock();
+
+ FSDirectory fsd = fsn.getFSDirectory();
+ byte[][] pathComponents = FSDirectory
+ .getPathComponentsForReservedPath(srcArg);
+ final String src;
+ final INodesInPath iip;
+ final boolean onBlockBoundary;
+ Block truncateBlock = null;
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, srcArg, pathComponents);
+ iip = fsd.getINodesInPath4Write(src, true);
+ if (fsn.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+ INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
+ final BlockStoragePolicy lpPolicy = fsn.getBlockManager()
+ .getStoragePolicy("LAZY_PERSIST");
+
+ if (lpPolicy != null && lpPolicy.getId() == file.getStoragePolicyID()) {
+ throw new UnsupportedOperationException(
+ "Cannot truncate lazy persist file " + src);
+ }
+
+ // Check if the file is already being truncated with the same length
+ final BlockInfo last = file.getLastBlock();
+ if (last != null && last.getBlockUCState()
+ == BlockUCState.UNDER_RECOVERY) {
+ final Block truncatedBlock = ((BlockInfoUnderConstruction) last)
+ .getTruncateBlock();
+ if (truncatedBlock != null) {
+ final long truncateLength = file.computeFileSize(false, false)
+ + truncatedBlock.getNumBytes();
+ if (newLength == truncateLength) {
+ return new TruncateResult(false, fsd.getAuditFileInfo(iip));
+ }
+ }
+ }
+
+ // Opening an existing file for truncate. May need lease recovery.
+ fsn.recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE, iip, src,
+ clientName, clientMachine, false);
+ // Truncate length check.
+ long oldLength = file.computeFileSize();
+ if (oldLength == newLength) {
+ return new TruncateResult(true, fsd.getAuditFileInfo(iip));
+ }
+ if (oldLength < newLength) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot truncate to a larger file size. Current size: " + oldLength
+ + ", truncate size: " + newLength + ".");
+ }
+ // Perform INodeFile truncation.
+ final QuotaCounts delta = new QuotaCounts.Builder().build();
+ onBlockBoundary = unprotectedTruncate(fsn, iip, newLength,
+ toRemoveBlocks, mtime, delta);
+ if (!onBlockBoundary) {
+ // Open file for write, but don't log into edits
+ long lastBlockDelta = file.computeFileSize() - newLength;
+ assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
+ truncateBlock = prepareFileForTruncate(fsn, iip, clientName,
+ clientMachine, lastBlockDelta, null);
+ }
+
+ // update the quota: use the preferred block size for UC block
+ fsd.updateCountNoQuotaCheck(iip, iip.length() - 1, delta);
+ } finally {
+ fsd.writeUnlock();
+ }
+
+ fsn.getEditLog().logTruncate(src, clientName, clientMachine, newLength,
+ mtime, truncateBlock);
+ return new TruncateResult(onBlockBoundary, fsd.getAuditFileInfo(iip));
+ }
+
+ /**
+ * Unprotected truncate implementation. Unlike
+ * {@link FSDirTruncateOp#truncate}, this will not schedule block recovery.
+ *
+ * @param fsn namespace
+ * @param src path name
+ * @param clientName client name
+ * @param clientMachine client machine info
+ * @param newLength the target file size
+ * @param mtime modified time
+ * @param truncateBlock truncate block
+ * @throws IOException
+ */
+ static void unprotectedTruncate(final FSNamesystem fsn, final String src,
+ final String clientName, final String clientMachine,
+ final long newLength, final long mtime, final Block truncateBlock)
+ throws UnresolvedLinkException, QuotaExceededException,
+ SnapshotAccessControlException, IOException {
+ assert fsn.hasWriteLock();
+
+ FSDirectory fsd = fsn.getFSDirectory();
+ INodesInPath iip = fsd.getINodesInPath(src, true);
+ INodeFile file = iip.getLastINode().asFile();
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ boolean onBlockBoundary = unprotectedTruncate(fsn, iip, newLength,
+ collectedBlocks, mtime, null);
+
+ if (!onBlockBoundary) {
+ BlockInfo oldBlock = file.getLastBlock();
+ Block tBlk = prepareFileForTruncate(fsn, iip, clientName, clientMachine,
+ file.computeFileSize() - newLength, truncateBlock);
+ assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
+ tBlk.getNumBytes() == truncateBlock.getNumBytes() :
+ "Should be the same block.";
+ if (oldBlock.getBlockId() != tBlk.getBlockId()
+ && !file.isBlockInLatestSnapshot(oldBlock)) {
+ fsn.getBlockManager().removeBlockFromMap(oldBlock);
+ }
+ }
+ assert onBlockBoundary == (truncateBlock == null) :
+ "truncateBlock is null iff on block boundary: " + truncateBlock;
+ fsn.removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+ }
+
+ /**
+ * Convert current INode to UnderConstruction. Recreate lease. Create new
+ * block for the truncated copy. Schedule truncation of the replicas.
+ *
+ * @param fsn namespace
+ * @param iip inodes in the path containing the file
+ * @param leaseHolder lease holder
+ * @param clientMachine client machine info
+ * @param lastBlockDelta last block delta size
+ * @param newBlock new block
+ * @return the returned block will be written to editLog and passed back
+ * into this method upon loading.
+ * @throws IOException
+ */
+ @VisibleForTesting
+ static Block prepareFileForTruncate(FSNamesystem fsn, INodesInPath iip,
+ String leaseHolder, String clientMachine, long lastBlockDelta,
+ Block newBlock) throws IOException {
+ assert fsn.hasWriteLock();
+
+ INodeFile file = iip.getLastINode().asFile();
+ file.recordModification(iip.getLatestSnapshotId());
+ file.toUnderConstruction(leaseHolder, clientMachine);
+ assert file.isUnderConstruction() : "inode should be under construction.";
+ fsn.getLeaseManager().addLease(
+ file.getFileUnderConstructionFeature().getClientName(), file.getId());
+ boolean shouldRecoverNow = (newBlock == null);
+ BlockInfo oldBlock = file.getLastBlock();
+ boolean shouldCopyOnTruncate = shouldCopyOnTruncate(fsn, file, oldBlock);
+ if (newBlock == null) {
+ newBlock = (shouldCopyOnTruncate) ? fsn.createNewBlock() : new Block(
+ oldBlock.getBlockId(), oldBlock.getNumBytes(),
+ fsn.nextGenerationStamp(fsn.getBlockIdManager().isLegacyBlock(
+ oldBlock)));
+ }
+
+ BlockInfoUnderConstruction truncatedBlockUC;
+ if (shouldCopyOnTruncate) {
+ // Add new truncateBlock into blocksMap and
+ // use oldBlock as a source for copy-on-truncate recovery
+ truncatedBlockUC = new BlockInfoUnderConstructionContiguous(newBlock,
+ file.getPreferredBlockReplication());
+ truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
+ truncatedBlockUC.setTruncateBlock(oldBlock);
+ file.setLastBlock(truncatedBlockUC,
+ fsn.getBlockManager().getStorages(oldBlock));
+ fsn.getBlockManager().addBlockCollection(truncatedBlockUC, file);
+
+ NameNode.stateChangeLog.debug(
+ "BLOCK* prepareFileForTruncate: Scheduling copy-on-truncate to new"
+ + " size {} new block {} old block {}",
+ truncatedBlockUC.getNumBytes(), newBlock,
+ truncatedBlockUC.getTruncateBlock());
+ } else {
+ // Use new generation stamp for in-place truncate recovery
+ fsn.getBlockManager().convertLastBlockToUnderConstruction(file,
+ lastBlockDelta);
+ oldBlock = file.getLastBlock();
+ assert !oldBlock.isComplete() : "oldBlock should be under construction";
+ truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
+ truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
+ truncatedBlockUC.getTruncateBlock().setNumBytes(
+ oldBlock.getNumBytes() - lastBlockDelta);
+ truncatedBlockUC.getTruncateBlock().setGenerationStamp(
+ newBlock.getGenerationStamp());
+
+ NameNode.stateChangeLog.debug(
+ "BLOCK* prepareFileForTruncate: {} Scheduling in-place block "
+ + "truncate to new size {}", truncatedBlockUC.getTruncateBlock()
+ .getNumBytes(), truncatedBlockUC);
+ }
+ if (shouldRecoverNow) {
+ truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
+ }
+
+ return newBlock;
+ }
+
+ /**
+ * Truncate has the following properties:
+ * 1.) Any block deletions occur now.
+ * 2.) INode length is truncated now - new clients can only read up to
+ * the truncated length.
+ * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
+ * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
+ * 5.) File is considered UNDER_RECOVERY until truncation recovery
+ * completes.
+ * 6.) Soft and hard Lease expiration require truncation recovery to
+ * complete.
+ *
+ * @return true if on the block boundary or false if recovery is need
+ */
+ private static boolean unprotectedTruncate(FSNamesystem fsn,
+ INodesInPath iip, long newLength, BlocksMapUpdateInfo collectedBlocks,
+ long mtime, QuotaCounts delta) throws IOException {
+ assert fsn.hasWriteLock();
+
+ INodeFile file = iip.getLastINode().asFile();
+ int latestSnapshot = iip.getLatestSnapshotId();
+ file.recordModification(latestSnapshot, true);
+
+ verifyQuotaForTruncate(fsn, iip, file, newLength, delta);
+
+ long remainingLength =
+ file.collectBlocksBeyondMax(newLength, collectedBlocks);
+ file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
+ file.setModificationTime(mtime);
+ // return whether on a block boundary
+ return (remainingLength - newLength) == 0;
+ }
+
+ private static void verifyQuotaForTruncate(FSNamesystem fsn,
+ INodesInPath iip, INodeFile file, long newLength, QuotaCounts delta)
+ throws QuotaExceededException {
+ FSDirectory fsd = fsn.getFSDirectory();
+ if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
+ // Do not check quota if edit log is still being processed
+ return;
+ }
+ final BlockStoragePolicy policy = fsd.getBlockStoragePolicySuite()
+ .getPolicy(file.getStoragePolicyID());
+ file.computeQuotaDeltaForTruncate(newLength, policy, delta);
+ fsd.readLock();
+ try {
+ FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /**
+ * Defines if a replica needs to be copied on truncate or
+ * can be truncated in place.
+ */
+ private static boolean shouldCopyOnTruncate(FSNamesystem fsn, INodeFile file,
+ BlockInfo blk) {
+ if (!fsn.isUpgradeFinalized()) {
+ return true;
+ }
+ if (fsn.isRollingUpgrade()) {
+ return true;
+ }
+ return file.isBlockInLatestSnapshot(blk);
+ }
+
+ /**
+ * Result of truncate operation.
+ */
+ static class TruncateResult {
+ private final boolean result;
+ private final HdfsFileStatus stat;
+
+ public TruncateResult(boolean result, HdfsFileStatus stat) {
+ this.result = result;
+ this.stat = stat;
+ }
+
+ /**
+ * @return true if client does not need to wait for block recovery,
+ * false if client needs to wait for block recovery.
+ */
+ boolean getResult() {
+ return result;
+ }
+
+ /**
+ * @return file information.
+ */
+ HdfsFileStatus getFileStatus() {
+ return stat;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c807fba..ccee1ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
@@ -49,11 +48,9 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.util.ByteArray;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.security.AccessControlException;
@@ -908,98 +905,6 @@ public class FSDirectory implements Closeable {
}
/**
- * FSEditLogLoader implementation.
- * Unlike FSNamesystem.truncate, this will not schedule block recovery.
- */
- void unprotectedTruncate(String src, String clientName, String clientMachine,
- long newLength, long mtime, Block truncateBlock)
- throws UnresolvedLinkException, QuotaExceededException,
- SnapshotAccessControlException, IOException {
- INodesInPath iip = getINodesInPath(src, true);
- INodeFile file = iip.getLastINode().asFile();
- BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- boolean onBlockBoundary =
- unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null);
-
- if(! onBlockBoundary) {
- BlockInfo oldBlock = file.getLastBlock();
- Block tBlk =
- getFSNamesystem().prepareFileForTruncate(iip,
- clientName, clientMachine, file.computeFileSize() - newLength,
- truncateBlock);
- assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
- tBlk.getNumBytes() == truncateBlock.getNumBytes() :
- "Should be the same block.";
- if(oldBlock.getBlockId() != tBlk.getBlockId() &&
- !file.isBlockInLatestSnapshot(oldBlock)) {
- getBlockManager().removeBlockFromMap(oldBlock);
- }
- }
- assert onBlockBoundary == (truncateBlock == null) :
- "truncateBlock is null iff on block boundary: " + truncateBlock;
- getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
- }
-
- boolean truncate(INodesInPath iip, long newLength,
- BlocksMapUpdateInfo collectedBlocks,
- long mtime, QuotaCounts delta)
- throws IOException {
- writeLock();
- try {
- return unprotectedTruncate(iip, newLength, collectedBlocks, mtime, delta);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Truncate has the following properties:
- * 1.) Any block deletions occur now.
- * 2.) INode length is truncated now – new clients can only read up to
- * the truncated length.
- * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
- * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
- * 5.) File is considered UNDER_RECOVERY until truncation recovery completes.
- * 6.) Soft and hard Lease expiration require truncation recovery to complete.
- *
- * @return true if on the block boundary or false if recovery is need
- */
- boolean unprotectedTruncate(INodesInPath iip, long newLength,
- BlocksMapUpdateInfo collectedBlocks,
- long mtime, QuotaCounts delta) throws IOException {
- assert hasWriteLock();
- INodeFile file = iip.getLastINode().asFile();
- int latestSnapshot = iip.getLatestSnapshotId();
- file.recordModification(latestSnapshot, true);
-
- verifyQuotaForTruncate(iip, file, newLength, delta);
-
- long remainingLength =
- file.collectBlocksBeyondMax(newLength, collectedBlocks);
- file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
- file.setModificationTime(mtime);
- // return whether on a block boundary
- return (remainingLength - newLength) == 0;
- }
-
- private void verifyQuotaForTruncate(INodesInPath iip, INodeFile file,
- long newLength, QuotaCounts delta) throws QuotaExceededException {
- if (!getFSNamesystem().isImageLoaded() || shouldSkipQuotaChecks()) {
- // Do not check quota if edit log is still being processed
- return;
- }
- final BlockStoragePolicy policy = getBlockStoragePolicySuite()
- .getPolicy(file.getStoragePolicyID());
- file.computeQuotaDeltaForTruncate(newLength, policy, delta);
- readLock();
- try {
- verifyQuota(iip, iip.length() - 1, delta, null);
- } finally {
- readUnlock();
- }
- }
-
- /**
* This method is always called with writeLock of FSDirectory held.
*/
public final void addToInodeMap(INode inode) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index df01edd..63ef985 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -901,9 +901,9 @@ public class FSEditLogLoader {
}
case OP_TRUNCATE: {
TruncateOp truncateOp = (TruncateOp) op;
- fsDir.unprotectedTruncate(truncateOp.src, truncateOp.clientName,
- truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp,
- truncateOp.truncateBlock);
+ FSDirTruncateOp.unprotectedTruncate(fsNamesys, truncateOp.src,
+ truncateOp.clientName, truncateOp.clientMachine,
+ truncateOp.newLength, truncateOp.timestamp, truncateOp.truncateBlock);
break;
}
case OP_SET_STORAGE_POLICY: {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e95007b..7c6d6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -201,7 +201,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -1831,218 +1830,44 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* block recovery to truncate the last block of the file.
*
* @return true if client does not need to wait for block recovery,
- * false if client needs to wait for block recovery.
+ * false if client needs to wait for block recovery.
*/
- boolean truncate(String src, long newLength,
- String clientName, String clientMachine,
- long mtime)
- throws IOException, UnresolvedLinkException {
+ boolean truncate(String src, long newLength, String clientName,
+ String clientMachine, long mtime) throws IOException,
+ UnresolvedLinkException {
+
requireEffectiveLayoutVersionForFeature(Feature.TRUNCATE);
- boolean ret;
+ final FSDirTruncateOp.TruncateResult r;
try {
- ret = truncateInt(src, newLength, clientName, clientMachine, mtime);
+ NameNode.stateChangeLog.debug(
+ "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
+ if (newLength < 0) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot truncate to a negative file size: " + newLength + ".");
+ }
+ final FSPermissionChecker pc = getPermissionChecker();
+ checkOperation(OperationCategory.WRITE);
+ writeLock();
+ BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot truncate for " + src);
+ r = FSDirTruncateOp.truncate(this, src, newLength, clientName,
+ clientMachine, mtime, toRemoveBlocks, pc);
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
+ removeBlocks(toRemoveBlocks);
+ toRemoveBlocks.clear();
+ }
+ logAuditEvent(true, "truncate", src, null, r.getFileStatus());
} catch (AccessControlException e) {
logAuditEvent(false, "truncate", src);
throw e;
}
- return ret;
- }
-
- boolean truncateInt(String srcArg, long newLength,
- String clientName, String clientMachine,
- long mtime)
- throws IOException, UnresolvedLinkException {
- String src = srcArg;
- NameNode.stateChangeLog.debug(
- "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
- if (newLength < 0) {
- throw new HadoopIllegalArgumentException(
- "Cannot truncate to a negative file size: " + newLength + ".");
- }
- HdfsFileStatus stat = null;
- FSPermissionChecker pc = getPermissionChecker();
- checkOperation(OperationCategory.WRITE);
- boolean res;
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- writeLock();
- BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
- try {
- checkOperation(OperationCategory.WRITE);
- checkNameNodeSafeMode("Cannot truncate for " + src);
- src = dir.resolvePath(pc, src, pathComponents);
- res = truncateInternal(src, newLength, clientName,
- clientMachine, mtime, pc, toRemoveBlocks);
- stat = dir.getAuditFileInfo(dir.getINodesInPath4Write(src, false));
- } finally {
- writeUnlock();
- }
- getEditLog().logSync();
- if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
- removeBlocks(toRemoveBlocks);
- toRemoveBlocks.clear();
- }
- logAuditEvent(true, "truncate", src, null, stat);
- return res;
- }
-
- /**
- * Truncate a file to a given size
- * Update the count at each ancestor directory with quota
- */
- boolean truncateInternal(String src, long newLength,
- String clientName, String clientMachine,
- long mtime, FSPermissionChecker pc,
- BlocksMapUpdateInfo toRemoveBlocks)
- throws IOException, UnresolvedLinkException {
- assert hasWriteLock();
- INodesInPath iip = dir.getINodesInPath4Write(src, true);
- if (isPermissionEnabled) {
- dir.checkPathAccess(pc, iip, FsAction.WRITE);
- }
- INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
- final BlockStoragePolicy lpPolicy =
- blockManager.getStoragePolicy("LAZY_PERSIST");
-
- if (lpPolicy != null &&
- lpPolicy.getId() == file.getStoragePolicyID()) {
- throw new UnsupportedOperationException(
- "Cannot truncate lazy persist file " + src);
- }
-
- // Check if the file is already being truncated with the same length
- final BlockInfo last = file.getLastBlock();
- if (last != null && last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
- final Block truncateBlock
- = ((BlockInfoUnderConstruction)last).getTruncateBlock();
- if (truncateBlock != null) {
- final long truncateLength = file.computeFileSize(false, false)
- + truncateBlock.getNumBytes();
- if (newLength == truncateLength) {
- return false;
- }
- }
- }
-
- // Opening an existing file for truncate. May need lease recovery.
- recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE,
- iip, src, clientName, clientMachine, false);
- // Truncate length check.
- long oldLength = file.computeFileSize();
- if(oldLength == newLength) {
- return true;
- }
- if(oldLength < newLength) {
- throw new HadoopIllegalArgumentException(
- "Cannot truncate to a larger file size. Current size: " + oldLength +
- ", truncate size: " + newLength + ".");
- }
- // Perform INodeFile truncation.
- final QuotaCounts delta = new QuotaCounts.Builder().build();
- boolean onBlockBoundary = dir.truncate(iip, newLength, toRemoveBlocks,
- mtime, delta);
- Block truncateBlock = null;
- if(!onBlockBoundary) {
- // Open file for write, but don't log into edits
- long lastBlockDelta = file.computeFileSize() - newLength;
- assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
- truncateBlock = prepareFileForTruncate(iip, clientName, clientMachine,
- lastBlockDelta, null);
- }
-
- // update the quota: use the preferred block size for UC block
- dir.writeLock();
- try {
- dir.updateCountNoQuotaCheck(iip, iip.length() - 1, delta);
- } finally {
- dir.writeUnlock();
- }
-
- getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime,
- truncateBlock);
- return onBlockBoundary;
- }
-
- /**
- * Convert current INode to UnderConstruction.
- * Recreate lease.
- * Create new block for the truncated copy.
- * Schedule truncation of the replicas.
- *
- * @return the returned block will be written to editLog and passed back into
- * this method upon loading.
- */
- Block prepareFileForTruncate(INodesInPath iip,
- String leaseHolder,
- String clientMachine,
- long lastBlockDelta,
- Block newBlock)
- throws IOException {
- INodeFile file = iip.getLastINode().asFile();
- file.recordModification(iip.getLatestSnapshotId());
- file.toUnderConstruction(leaseHolder, clientMachine);
- assert file.isUnderConstruction() : "inode should be under construction.";
- leaseManager.addLease(
- file.getFileUnderConstructionFeature().getClientName(), file.getId());
- boolean shouldRecoverNow = (newBlock == null);
- BlockInfo oldBlock = file.getLastBlock();
- boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
- if(newBlock == null) {
- newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
- new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
- nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock)));
- }
-
- BlockInfoUnderConstruction truncatedBlockUC;
- if(shouldCopyOnTruncate) {
- // Add new truncateBlock into blocksMap and
- // use oldBlock as a source for copy-on-truncate recovery
- truncatedBlockUC = new BlockInfoUnderConstructionContiguous(newBlock,
- file.getPreferredBlockReplication());
- truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
- truncatedBlockUC.setTruncateBlock(oldBlock);
- file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock));
- getBlockManager().addBlockCollection(truncatedBlockUC, file);
-
- NameNode.stateChangeLog.debug(
- "BLOCK* prepareFileForTruncate: Scheduling copy-on-truncate to new" +
- " size {} new block {} old block {}", truncatedBlockUC.getNumBytes(),
- newBlock, truncatedBlockUC.getTruncateBlock());
- } else {
- // Use new generation stamp for in-place truncate recovery
- blockManager.convertLastBlockToUnderConstruction(file, lastBlockDelta);
- oldBlock = file.getLastBlock();
- assert !oldBlock.isComplete() : "oldBlock should be under construction";
- truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
- truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
- truncatedBlockUC.getTruncateBlock().setNumBytes(
- oldBlock.getNumBytes() - lastBlockDelta);
- truncatedBlockUC.getTruncateBlock().setGenerationStamp(
- newBlock.getGenerationStamp());
-
- NameNode.stateChangeLog.debug(
- "BLOCK* prepareFileForTruncate: {} Scheduling in-place block " +
- "truncate to new size {}",
- truncatedBlockUC.getTruncateBlock().getNumBytes(), truncatedBlockUC);
- }
- if (shouldRecoverNow) {
- truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
- }
-
- return newBlock;
- }
-
- /**
- * Defines if a replica needs to be copied on truncate or
- * can be truncated in place.
- */
- boolean shouldCopyOnTruncate(INodeFile file, BlockInfo blk) {
- if(!isUpgradeFinalized()) {
- return true;
- }
- if (isRollingUpgrade()) {
- return true;
- }
- return file.isBlockInLatestSnapshot(blk);
+ return r.getResult();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index df920e0..e0f9ad2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -1008,8 +1008,8 @@ public class TestFileTruncate {
fsn.writeLock();
try {
Block oldBlock = file.getLastBlock();
- Block truncateBlock =
- fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+ Block truncateBlock = FSDirTruncateOp.prepareFileForTruncate(fsn, iip,
+ client, clientMachine, 1, null);
// In-place truncate uses old block id with new genStamp.
assertThat(truncateBlock.getBlockId(),
is(equalTo(oldBlock.getBlockId())));
@@ -1041,8 +1041,8 @@ public class TestFileTruncate {
fsn.writeLock();
try {
Block oldBlock = file.getLastBlock();
- Block truncateBlock =
- fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+ Block truncateBlock = FSDirTruncateOp.prepareFileForTruncate(fsn, iip,
+ client, clientMachine, 1, null);
// Copy-on-write truncate makes new block with new id and genStamp
assertThat(truncateBlock.getBlockId(),
is(not(equalTo(oldBlock.getBlockId()))));
[18/48] hadoop git commit: HDFS-8635. Migrate HDFS native build to
new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by zj...@apache.org.
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b402281
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b402281
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b402281
Branch: refs/heads/YARN-2928
Commit: 5b4022815ed8ce2b2777388682474fa775849881
Parents: 8ed2be0
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 15:54:38 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop-hdfs/src/CMakeLists.txt | 129 ++++++-------------
.../src/contrib/libwebhdfs/CMakeLists.txt | 8 +-
.../libwebhdfs/resources/FindJansson.cmake | 9 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 91 +++++++------
5 files changed, 100 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b065f98..65f286d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8493. Consolidate truncate() related implementation in a single class.
(Rakesh R via wheat9)
+ HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
+ via Colin P. McCabe)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 563727b..b4a3b40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -18,106 +18,59 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
- add_library(${LIBNAME} SHARED ${ARGN})
- add_library(${LIBNAME}_static STATIC ${ARGN})
- # Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so
- # (dynamic linking). On Windows, we cannot use the same base name for both
- # static and dynamic, because Windows does not use distinct file extensions
- # for a statically linked library vs. a DLL import library. Both use the
- # .lib extension. On Windows, we'll build the static library as
- # hdfs_static.lib.
- if (NOT WIN32)
- set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
- endif (NOT WIN32)
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
- target_link_libraries(${LIBNAME} ${ARGN})
- target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
- output_directory(${TGT} "${DIR}")
- output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
+#
+# Main configuration
+#
-# Flatten a list into a string.
-function(FLATTEN_LIST INPUT SEPARATOR OUTPUT)
- string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
- set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
-endfunction()
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+ message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
# Check to see if our compiler and linker support the __thread attribute.
-# On Linux and some other operating systems, this is a more efficient
-# alternative to POSIX thread local storage.
-INCLUDE(CheckCSourceCompiles)
-CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+# On Linux and some other operating systems, this is a more efficient
+# alternative to POSIX thread local storage.
+include(CheckCSourceCompiles)
+check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
# Check to see if we have Intel SSE intrinsics.
-CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
+check_c_source_compileS("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
# Check if we need to link dl library to get dlopen.
# dlopen on Linux is in separate library but on FreeBSD its in libc
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
-find_package(JNI REQUIRED)
-if (NOT GENERATED_JAVAH)
- # Must identify where the generated headers have been placed
- MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-
-if (WIN32)
+if(WIN32)
+ # Set the optimizer level.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2")
-
# Set warning level 4.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
-
# Skip "unreferenced formal parameter".
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100")
-
# Skip "conditional expression is constant".
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127")
-
# Skip deprecated POSIX function warnings.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE")
-
# Skip CRT non-secure function warnings. If we can convert usage of
# strerror, getenv and ctime to their secure CRT equivalents, then we can
# re-enable the CRT non-secure function warnings.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS")
-
# Omit unneeded headers.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
-
set(OS_DIR main/native/libhdfs/os/windows)
set(OUT_DIR target/bin)
-else (WIN32)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
+else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(OS_DIR main/native/libhdfs/os/posix)
- set(OS_LINK_LIBRARIES pthread)
set(OUT_DIR target/usr/local/lib)
-endif (WIN32)
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
add_definitions(-DLIBHDFS_DLL_EXPORT)
@@ -132,9 +85,9 @@ include_directories(
)
set(_FUSE_DFS_VERSION 0.1.0)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
-add_dual_library(hdfs
+hadoop_add_dual_library(hdfs
main/native/libhdfs/exception.c
main/native/libhdfs/jni_helper.c
main/native/libhdfs/hdfs.c
@@ -142,17 +95,17 @@ add_dual_library(hdfs
${OS_DIR}/mutexes.c
${OS_DIR}/thread_local_storage.c
)
-if (NEED_LINK_DL)
+if(NEED_LINK_DL)
set(LIB_DL dl)
-endif(NEED_LINK_DL)
+endif()
-target_link_dual_libraries(hdfs
+hadoop_target_link_dual_libraries(hdfs
${JAVA_JVM_LIBRARY}
${LIB_DL}
${OS_LINK_LIBRARIES}
)
-dual_output_directory(hdfs ${OUT_DIR})
+hadoop_dual_output_directory(hdfs ${OUT_DIR})
set(LIBHDFS_VERSION "0.0.0")
set_target_properties(hdfs PROPERTIES
SOVERSION ${LIBHDFS_VERSION})
@@ -232,25 +185,21 @@ target_link_libraries(test_htable
# Skip vecsum on Windows. This could be made to work in the future by
# introducing an abstraction layer over the sys/mman.h functions.
-if (NOT WIN32)
- add_executable(test_libhdfs_vecsum
- main/native/libhdfs/test/vecsum.c
- )
- if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(NOT WIN32)
+ add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c)
+ if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(test_libhdfs_vecsum
hdfs
- pthread
- )
- else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ pthread)
+ else()
target_link_libraries(test_libhdfs_vecsum
hdfs
pthread
- rt
- )
- endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-endif(NOT WIN32)
+ rt)
+ endif()
+endif()
-IF(REQUIRE_LIBWEBHDFS)
+if(REQUIRE_LIBWEBHDFS)
add_subdirectory(contrib/libwebhdfs)
-ENDIF(REQUIRE_LIBWEBHDFS)
+endif()
add_subdirectory(main/native/fuse-dfs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
index 44f0c42..dc74feb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
@@ -19,12 +19,12 @@
find_package(CURL REQUIRED)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
+ "${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
find_package(Jansson REQUIRED)
include_directories(${JANSSON_INCLUDE_DIR})
-add_dual_library(webhdfs
+hadoop_add_dual_library(webhdfs
src/hdfs_web.c
src/hdfs_http_client.c
src/hdfs_http_query.c
@@ -35,13 +35,13 @@ add_dual_library(webhdfs
../../${OS_DIR}/thread_local_storage.c
../../main/native/libhdfs/common/htable.c
)
-target_link_dual_libraries(webhdfs
+hadoop_target_link_dual_libraries(webhdfs
${JAVA_JVM_LIBRARY}
${CURL_LIBRARY}
${JANSSON_LIBRARY}
pthread
)
-dual_output_directory(webhdfs target)
+hadoop_dual_output_directory(webhdfs target)
set(LIBWEBHDFS_VERSION "0.0.0")
set_target_properties(webhdfs PROPERTIES
SOVERSION ${LIBWEBHDFS_VERSION})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
index cb8c8db..b8c67ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
@@ -5,9 +5,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,8 @@
# JANSSON_DEFINITIONS - Compiler switches required for using Jansson
find_path(JANSSON_INCLUDE_DIR jansson.h
- /usr/incluce
+ /usr/include
+ /usr/include/jansson
/usr/local/include )
find_library(JANSSON_LIBRARY NAMES jansson
@@ -39,4 +40,4 @@ include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Jansson DEFAULT_MSG
JANSSON_LIBRARY JANSSON_INCLUDE_DIR)
-mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
\ No newline at end of file
+mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
index 8a8ea5b..7a5e5a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
@@ -18,57 +18,63 @@
set(CMAKE_SKIP_RPATH TRUE)
+# Flatten a list into a string.
+function(flatten_list INPUT SEPARATOR OUTPUT)
+ string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
+ set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
+endfunction()
+
# Find Linux FUSE
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
find_package(PkgConfig REQUIRED)
pkg_check_modules(FUSE fuse)
- IF(FUSE_FOUND)
- FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
- FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
+ if(FUSE_FOUND)
+ flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
+ flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}")
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}")
- MESSAGE(STATUS "Building Linux FUSE client.")
+ message(STATUS "Building Linux FUSE client.")
include_directories(${FUSE_INCLUDE_DIRS})
- ELSE(FUSE_FOUND)
- MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
- ENDIF(FUSE_FOUND)
-ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.")
-ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+ else()
+ message(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
+ endif(FUSE_FOUND)
+else()
+ message(STATUS "Non-Linux system detected. Will not build FUSE client.")
+endif()
-IF(FUSE_FOUND)
+if(FUSE_FOUND)
add_library(posix_util
../util/posix_util.c
)
add_executable(fuse_dfs
fuse_dfs.c
- fuse_options.c
- fuse_connect.c
- fuse_impls_access.c
- fuse_impls_chmod.c
- fuse_impls_chown.c
- fuse_impls_create.c
- fuse_impls_flush.c
- fuse_impls_getattr.c
- fuse_impls_mkdir.c
- fuse_impls_mknod.c
- fuse_impls_open.c
- fuse_impls_read.c
- fuse_impls_readdir.c
- fuse_impls_release.c
- fuse_impls_rename.c
- fuse_impls_rmdir.c
- fuse_impls_statfs.c
- fuse_impls_symlink.c
- fuse_impls_truncate.c
- fuse_impls_unlink.c
- fuse_impls_utimens.c
+ fuse_options.c
+ fuse_connect.c
+ fuse_impls_access.c
+ fuse_impls_chmod.c
+ fuse_impls_chown.c
+ fuse_impls_create.c
+ fuse_impls_flush.c
+ fuse_impls_getattr.c
+ fuse_impls_mkdir.c
+ fuse_impls_mknod.c
+ fuse_impls_open.c
+ fuse_impls_read.c
+ fuse_impls_readdir.c
+ fuse_impls_release.c
+ fuse_impls_rename.c
+ fuse_impls_rmdir.c
+ fuse_impls_statfs.c
+ fuse_impls_symlink.c
+ fuse_impls_truncate.c
+ fuse_impls_unlink.c
+ fuse_impls_utimens.c
fuse_impls_write.c
- fuse_init.c
- fuse_stat_struct.c
- fuse_trash.c
- fuse_users.c
+ fuse_init.c
+ fuse_stat_struct.c
+ fuse_trash.c
+ fuse_users.c
)
target_link_libraries(fuse_dfs
${FUSE_LIBRARIES}
@@ -88,8 +94,9 @@ IF(FUSE_FOUND)
posix_util
pthread
)
-ELSE(FUSE_FOUND)
- IF(REQUIRE_FUSE)
- MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.")
- ENDIF(REQUIRE_FUSE)
-ENDIF(FUSE_FOUND)
+ set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning
+else()
+ if(REQUIRE_FUSE)
+ message(FATAL_ERROR "Required component fuse_dfs could not be built.")
+ endif()
+endif()
[31/48] hadoop git commit: MAPREDUCE-6420. Interrupted Exception in
LocalContainerLauncher should be logged in warn/info level. Contributed by
Chang Li
Posted by zj...@apache.org.
MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be logged in warn/info level. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef7b82be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef7b82be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef7b82be
Branch: refs/heads/YARN-2928
Commit: ef7b82be1281f1c64bab9bfc6d90ac334fe62868
Parents: d57f65e
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 17:50:02 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapred/LocalContainerLauncher.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef7b82be/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 49a873d..dd739ca 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
(rchiang via rkanter)
+ MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be
+ logged in warn/info level (Chang Li via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef7b82be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 9d8b4a5..1a0d5fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -233,7 +233,7 @@ public class LocalContainerLauncher extends AbstractService implements
try {
event = eventQueue.take();
} catch (InterruptedException e) { // mostly via T_KILL? JOB_KILL?
- LOG.error("Returning, interrupted : " + e);
+ LOG.warn("Returning, interrupted : " + e);
break;
}
[17/48] hadoop git commit: HADOOP-12158. Improve error message in
TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed.
Posted by zj...@apache.org.
HADOOP-12158. Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ed2be02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ed2be02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ed2be02
Branch: refs/heads/YARN-2928
Commit: 8ed2be02f9817bf3de136cb53ed23c016cfd067d
Parents: bcac9c9
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Jun 30 15:04:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++++
.../TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 9 +++++++--
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed2be02/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e332ea8..2aab8b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -666,6 +666,11 @@ Release 2.8.0 - UNRELEASED
HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
occurs (Jason Lowe via jeagles)
+
+ HADOOP-12158. Improve error message in
+ TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not
+ installed. (wang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed2be02/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index 4f90a0c..684ec09 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.junit.Assert;
import org.junit.BeforeClass;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
extends TestCryptoStreams {
@@ -32,7 +34,10 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
OpensslAesCtrCryptoCodec.class.getName());
codec = CryptoCodec.getInstance(conf);
- Assert.assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
+ assertNotNull("Unable to instantiate codec " +
+ OpensslAesCtrCryptoCodec.class.getName() + ", is the required "
+ + "version of OpenSSL installed?", codec);
+ assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
codec.getClass().getCanonicalName());
}
}
[05/48] hadoop git commit: HADOOP-12104. Migrate Hadoop Pipes native
build to new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by zj...@apache.org.
HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96d07d2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96d07d2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96d07d2d
Branch: refs/heads/YARN-2928
Commit: 96d07d2dbae3f1a30cc178ee237402369d034350
Parents: e9d1005
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 29 12:01:17 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 49 +++++++++-----------
2 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d07d2d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0a964a3..e8e85a0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash.
(wang)
+ HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework
+ (alanburlison via cmccabe)
+
HADOOP-12036. Consolidate all of the cmake extensions in one directory
(alanburlison via cmccabe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d07d2d/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
index 170af49..3b0b28c 100644
--- a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
+++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
@@ -6,7 +6,7 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-
+#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
@@ -17,25 +17,11 @@
#
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-find_package(OpenSSL REQUIRED)
-
-set(CMAKE_BUILD_TYPE, Release)
-set(PIPES_FLAGS "-g -Wall -O2 -D_REENTRANT -D_GNU_SOURCE")
-set(PIPES_FLAGS "${PIPES_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PIPES_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PIPES_FLAGS}")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
+find_package(OpenSSL REQUIRED)
include_directories(
main/native/utils/api
@@ -47,19 +33,19 @@ include_directories(
# Example programs
add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
target_link_libraries(wordcount-simple hadooppipes hadooputils)
-output_directory(wordcount-simple examples)
+hadoop_output_directory(wordcount-simple examples)
add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
target_link_libraries(wordcount-part hadooppipes hadooputils)
-output_directory(wordcount-part examples)
+hadoop_output_directory(wordcount-part examples)
add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
-output_directory(wordcount-nopipe examples)
+hadoop_output_directory(wordcount-nopipe examples)
add_executable(pipes-sort main/native/examples/impl/sort.cc)
target_link_libraries(pipes-sort hadooppipes hadooputils)
-output_directory(pipes-sort examples)
+hadoop_output_directory(pipes-sort examples)
add_library(hadooputils STATIC
main/native/utils/impl/StringUtils.cc
@@ -70,15 +56,22 @@ add_library(hadooppipes STATIC
main/native/pipes/impl/HadoopPipes.cc
)
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
+
+if(NEED_LINK_DL)
+ set(LIB_DL "dl")
+endif()
-if (NEED_LINK_DL)
- set(LIB_DL dl)
-endif (NEED_LINK_DL)
+if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+ exec_program("uname" ARGS "-r" OUTPUT_VARIABLE OS_VERSION)
+ if(OS_VERSION VERSION_LESS "5.12")
+ set(LIB_NET "socket" "nsl")
+ endif()
+endif()
target_link_libraries(hadooppipes
${OPENSSL_LIBRARIES}
${LIB_DL}
- pthread
+ ${LIB_NET}
)
[24/48] hadoop git commit: YARN-3827. Migrate YARN native build to
new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by zj...@apache.org.
YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e713f2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e713f2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e713f2d
Branch: refs/heads/YARN-2928
Commit: 0e713f2dc654667a9e7616bd0d4b573de07681ce
Parents: 71a07c6
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:24:19 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../src/CMakeLists.txt | 23 ++++++++++----------
2 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e713f2d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 896f54e..5269bec 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -420,6 +420,9 @@ Release 2.8.0 - UNRELEASED
YARN-3834. Scrub debug logging of tokens during resource localization.
(Chris Nauroth via xgong)
+ YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison
+ via Colin P. McCabe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e713f2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index f691a9c..d4d6ae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -16,25 +16,24 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
+# Note: can't use -D_FILE_OFFSET_BITS=64, see MAPREDUCE-4258
+string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
include(CheckFunctionExists)
-CHECK_FUNCTION_EXISTS(fcloseall HAVE_FCLOSEALL)
+check_function_exists(fcloseall HAVE_FCLOSEALL)
function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+ set_target_properties(${TGT} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+ set_target_properties(${TGT} PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+ set_target_properties(${TGT} PROPERTIES
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2 -D_GNU_SOURCE")
-# note: can't enable -D_LARGEFILE: see MAPREDUCE-4258
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT")
+endfunction()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}
@@ -42,7 +41,7 @@ include_directories(
main/native/container-executor
main/native/container-executor/impl
)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
add_library(container
main/native/container-executor/impl/configuration.c
[27/48] hadoop git commit: MAPREDUCE-6121.
JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via
rkanter)
Posted by zj...@apache.org.
MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/36a33ebd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/36a33ebd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/36a33ebd
Branch: refs/heads/YARN-2928
Commit: 36a33ebdb5a456279861275a052f57b7884700ad
Parents: c906166
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 30 16:46:41 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapreduce/JobResourceUploader.java | 38 +-------------------
2 files changed, 4 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a33ebd/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 5f4b3ef..d170b55 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host
(zhihai xu via jlowe)
+ MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
+ (rchiang via rkanter)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a33ebd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 134de35..f3e4d2f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -194,7 +194,7 @@ class JobResourceUploader {
FileSystem remoteFs = null;
remoteFs = originalPath.getFileSystem(conf);
- if (compareFs(remoteFs, jtFs)) {
+ if (FileUtil.compareFs(remoteFs, jtFs)) {
return originalPath;
}
// this might have name collisions. copy will throw an exception
@@ -205,42 +205,6 @@ class JobResourceUploader {
return newPath;
}
- /*
- * see if two file systems are the same or not.
- */
- private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
- URI srcUri = srcFs.getUri();
- URI dstUri = destFs.getUri();
- if (srcUri.getScheme() == null) {
- return false;
- }
- if (!srcUri.getScheme().equals(dstUri.getScheme())) {
- return false;
- }
- String srcHost = srcUri.getHost();
- String dstHost = dstUri.getHost();
- if ((srcHost != null) && (dstHost != null)) {
- try {
- srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
- dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
- } catch (UnknownHostException ue) {
- return false;
- }
- if (!srcHost.equals(dstHost)) {
- return false;
- }
- } else if (srcHost == null && dstHost != null) {
- return false;
- } else if (srcHost != null && dstHost == null) {
- return false;
- }
- // check for ports
- if (srcUri.getPort() != dstUri.getPort()) {
- return false;
- }
- return true;
- }
-
private void copyJar(Path originalJarPath, Path submitJarFile,
short replication) throws IOException {
jtFs.copyFromLocalFile(originalJarPath, submitJarFile);
[10/48] hadoop git commit: HDFS-8659. Block scanner INFO message is
spamming logs. Contributed by Yongjun Zhang.
Posted by zj...@apache.org.
HDFS-8659. Block scanner INFO message is spamming logs. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94dca164
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94dca164
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94dca164
Branch: refs/heads/YARN-2928
Commit: 94dca164d625670ab37f9d810a676f5a1ab36c38
Parents: 296d150
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Jun 29 14:35:39 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:54 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/datanode/BlockScanner.java | 2 +-
.../apache/hadoop/hdfs/server/datanode/VolumeScanner.java | 8 ++++----
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dca164/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0c56f2b..eb006eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -985,6 +985,8 @@ Release 2.7.2 - UNRELEASED
IMPROVEMENTS
+ HDFS-8659. Block scanner INFO message is spamming logs. (Yongjun Zhang)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dca164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index 9c4dd10..be6aa83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -317,7 +317,7 @@ public class BlockScanner {
*/
synchronized void markSuspectBlock(String storageId, ExtendedBlock block) {
if (!isEnabled()) {
- LOG.info("Not scanning suspicious block {} on {}, because the block " +
+ LOG.debug("Not scanning suspicious block {} on {}, because the block " +
"scanner is disabled.", block, storageId);
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dca164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 615abe9..ff655c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -656,24 +656,24 @@ public class VolumeScanner extends Thread {
public synchronized void markSuspectBlock(ExtendedBlock block) {
if (stopping) {
- LOG.info("{}: Not scheduling suspect block {} for " +
+ LOG.debug("{}: Not scheduling suspect block {} for " +
"rescanning, because this volume scanner is stopping.", this, block);
return;
}
Boolean recent = recentSuspectBlocks.getIfPresent(block);
if (recent != null) {
- LOG.info("{}: Not scheduling suspect block {} for " +
+ LOG.debug("{}: Not scheduling suspect block {} for " +
"rescanning, because we rescanned it recently.", this, block);
return;
}
if (suspectBlocks.contains(block)) {
- LOG.info("{}: suspect block {} is already queued for " +
+ LOG.debug("{}: suspect block {} is already queued for " +
"rescanning.", this, block);
return;
}
suspectBlocks.add(block);
recentSuspectBlocks.put(block, true);
- LOG.info("{}: Scheduling suspect block {} for rescanning.", this, block);
+ LOG.debug("{}: Scheduling suspect block {} for rescanning.", this, block);
notify(); // wake scanner thread.
}
[40/48] hadoop git commit: HDFS-8709. Clarify automatic sync in
FSEditLog#logEdit.
Posted by zj...@apache.org.
HDFS-8709. Clarify automatic sync in FSEditLog#logEdit.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0823b936
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0823b936
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0823b936
Branch: refs/heads/YARN-2928
Commit: 0823b9365e00c2341714e8f488ec9112f0e72eb7
Parents: 825dbb8
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Jul 2 10:26:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop/hdfs/server/namenode/FSEditLog.java | 28 +++++++++++++-------
2 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0823b936/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7b96c56..6678a3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -699,6 +699,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch
(vinayakumarb)
+ HDFS-8709. Clarify automatic sync in FSEditLog#logEdit. (wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0823b936/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 1b0b572..939e841 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -409,10 +409,14 @@ public class FSEditLog implements LogsPurgeable {
}
/**
- * Write an operation to the edit log. Do not sync to persistent
- * store yet.
+ * Write an operation to the edit log.
+ * <p/>
+ * Additionally, this will sync the edit log if required by the underlying
+ * edit stream's automatic sync policy (e.g. when the buffer is full, or
+ * if a time interval has elapsed).
*/
void logEdit(final FSEditLogOp op) {
+ boolean needsSync = false;
synchronized (this) {
assert isOpenForWrite() :
"bad state: " + state;
@@ -434,14 +438,16 @@ public class FSEditLog implements LogsPurgeable {
endTransaction(start);
// check if it is time to schedule an automatic sync
- if (!shouldForceSync()) {
- return;
+ needsSync = shouldForceSync();
+ if (needsSync) {
+ isAutoSyncScheduled = true;
}
- isAutoSyncScheduled = true;
}
- // sync buffered edit log entries to persistent store
- logSync();
+ // Sync the log if an automatic sync is required.
+ if (needsSync) {
+ logSync();
+ }
}
/**
@@ -1191,11 +1197,13 @@ public class FSEditLog implements LogsPurgeable {
throws IOException {
return journalSet.getEditLogManifest(fromTxId);
}
-
+
/**
* Finalizes the current edit log and opens a new log segment.
- * @return the transaction id of the BEGIN_LOG_SEGMENT transaction
- * in the new log.
+ *
+ * @param layoutVersion The layout version of the new edit log segment.
+ * @return the transaction id of the BEGIN_LOG_SEGMENT transaction in the new
+ * log.
*/
synchronized long rollEditLog(int layoutVersion) throws IOException {
LOG.info("Rolling edit logs");
[23/48] hadoop git commit: MAPREDUCE-6407. Migrate MAPREDUCE native
build to new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by zj...@apache.org.
MAPREDUCE-6407. Migrate MAPREDUCE native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a07c6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a07c6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a07c6e
Branch: refs/heads/YARN-2928
Commit: 71a07c6e1685a9bd1a4fb2fb6230b3201d9b7b76
Parents: 5b40228
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:06:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../src/CMakeLists.txt | 337 ++++++++-----------
.../src/JNIFlags.cmake | 118 -------
3 files changed, 137 insertions(+), 321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a07c6e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 974af61..5f4b3ef 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -139,6 +139,9 @@ Trunk (Unreleased)
MAPREDUCE-2632. Avoid calling the partitioner when the numReduceTasks is 1.
(Ravi Teja Ch N V and Sunil G via kasha)
+ MAPREDUCE-6407. Migrate MAPREDUCE nativetask build to new CMake framework
+ (Alan Burlison via Colin P. McCabe)
+
BUG FIXES
MAPREDUCE-6191. Improve clearing stale state of Java serialization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a07c6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 80036e8..f878a94 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -18,236 +18,170 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
-
-include(JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
- add_library(${LIBNAME} SHARED ${ARGN})
- add_library(${LIBNAME}_static STATIC ${ARGN})
- set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
- target_link_libraries(${LIBNAME} ${ARGN})
- target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
- output_directory(${TGT} "${DIR}")
- output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
-
-#
-# This macro alters the behavior of find_package and find_library.
-# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
-# You should save that variable before calling this function and restore it
-# after you have accomplished your goal.
-#
-# The behavior is altered in two ways:
-# 1. We always find shared libraries, never static;
-# 2. We find shared libraries with the given version number.
-#
-# On Windows this function is a no-op. Windows does not encode
-# version number information information into library path names.
-#
-macro(set_find_shared_library_version LVERS)
- IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
- # Mac OS uses .dylib
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib")
- ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
- # FreeBSD has always .so installed.
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
- ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
- # Windows doesn't support finding shared libraries by version.
- ELSE()
- # Most UNIX variants use .so
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}")
- ENDIF()
-endmacro(set_find_shared_library_version LVERS)
-
-if (NOT GENERATED_JAVAH)
- #Must identify where the generated headers have been placed
- MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-find_package(JNI REQUIRED)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-# primitive configs
-set(PRFLAGS "-DSIMPLE_MEMCPY")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -fno-strict-aliasing -Wall -Wno-sign-compare")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
- -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC")
-set(D main/native/)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-INCLUDE(CheckFunctionExists)
-INCLUDE(CheckCSourceCompiles)
-#INCLUDE(CheckLibraryExists)
-INCLUDE(CheckIncludeFiles)
-#CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
-#CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
-#CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
-CHECK_INCLUDE_FILES(fcntl.h HAVE_FCNTL_H)
-CHECK_INCLUDE_FILES(malloc.h HAVE_MALLOC_H)
-CHECK_INCLUDE_FILES(mach/mach.h HAVE_MACH_MACH_H)
-CHECK_INCLUDE_FILES(memory.h HAVE_MEMORY_H)
-CHECK_INCLUDE_FILES(stddef.h HAVE_STDDEF_H)
-CHECK_INCLUDE_FILES(stdint.h HAVE_STDINT_H)
-CHECK_INCLUDE_FILES(stdlib.h HAVE_STDLIB_H)
-CHECK_INCLUDE_FILES(string.h HAVE_STRING_H)
-CHECK_INCLUDE_FILES(unistd.h HAVE_UNITSTD_H)
-CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
-CHECK_FUNCTION_EXISTS(localtime_r HAVE_LOCALTIME_R)
-CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET)
-CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR)
-CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-common/)
+include(HadoopCommon)
+
+# Add extra compiler and linker flags.
+# -Wno-sign-compare
+hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing")
+
+# Source location.
+set(SRC main/native)
+
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+ message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
+
+# Probe for headers and functions.
+include(CheckFunctionExists)
+include(CheckIncludeFiles)
+
+check_include_files(fcntl.h HAVE_FCNTL_H)
+check_include_files(malloc.h HAVE_MALLOC_H)
+check_include_files(mach/mach.h HAVE_MACH_MACH_H)
+check_include_files(memory.h HAVE_MEMORY_H)
+check_include_files(stddef.h HAVE_STDDEF_H)
+check_include_files(stdint.h HAVE_STDINT_H)
+check_include_files(stdlib.h HAVE_STDLIB_H)
+Check_include_files(string.h HAVE_STRING_H)
+check_include_files(unistd.h HAVE_UNITSTD_H)
+check_function_exists(clock_gettime HAVE_CLOCK_GETTIME)
+check_function_exists(localtime_r HAVE_LOCALTIME_R)
+check_function_exists(memset HAVE_MEMSET)
+check_function_exists(strchr HAVE_STRCHR)
+check_function_exists(strtoul HAVE_STRTOUL)
+
+# Require snappy.
+set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+hadoop_set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
NAMES snappy
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+set(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
find_path(SNAPPY_INCLUDE_DIR
NAMES snappy.h
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
${CUSTOM_SNAPPY_INCLUDE})
-if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME)
set(SNAPPY_SOURCE_FILES
- "${D}/src/codec/SnappyCodec.cc")
-else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+ "${SRC}/src/codec/SnappyCodec.cc")
+ set(REQUIRE_SNAPPY ${REQUIRE_SNAPPY}) # Stop warning about unused variable.
+ message(STATUS "Found Snappy: ${SNAPPY_LIBRARY}")
+else()
set(SNAPPY_LIBRARY "")
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
- IF(REQUIRE_SNAPPY)
- MESSAGE(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
- ENDIF(REQUIRE_SNAPPY)
-endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+ if(REQUIRE_SNAPPY)
+ message(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
+ endif()
+endif()
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
include_directories(
${GENERATED_JAVAH}
- ${D}/src
- ${D}/src/util
- ${D}/src/lib
- ${D}/test
+ ${SRC}/src
+ ${SRC}/src/util
+ ${SRC}/src/lib
+ ${SRC}/test
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
# add gtest as system library to suppress gcc warnings
-include_directories(SYSTEM ${D}/gtest/include)
-
+include_directories(SYSTEM ${SRC}/gtest/include)
-SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
# macosx does not have -lrt
set(NT_DEPEND_LIBRARY dl pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
set(SYSTEM_MAC TRUE)
-else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+else()
set(NT_DEPEND_LIBRARY dl rt pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
set(SYSTEM_MAC FALSE)
-endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+endif()
configure_file(main/native/test.sh test/test.sh)
-add_dual_library(nativetask
+hadoop_add_dual_library(nativetask
${CMAKE_BINARY_DIR}/lz4.c
- ${D}/src/codec/BlockCodec.cc
- ${D}/src/codec/GzipCodec.cc
- ${D}/src/codec/Lz4Codec.cc
+ ${SRC}/src/codec/BlockCodec.cc
+ ${SRC}/src/codec/GzipCodec.cc
+ ${SRC}/src/codec/Lz4Codec.cc
${SNAPPY_SOURCE_FILES}
- ${D}/src/handler/BatchHandler.cc
- ${D}/src/handler/MCollectorOutputHandler.cc
- ${D}/src/handler/AbstractMapHandler.cc
- ${D}/src/handler/CombineHandler.cc
- ${D}/src/lib/Buffers.cc
- ${D}/src/lib/BufferStream.cc
- ${D}/src/lib/Compressions.cc
- ${D}/src/lib/PartitionBucket.cc
- ${D}/src/lib/PartitionBucketIterator.cc
- ${D}/src/lib/FileSystem.cc
- ${D}/src/lib/IFile.cc
- ${D}/src/lib/jniutils.cc
- ${D}/src/lib/Log.cc
- ${D}/src/lib/MapOutputCollector.cc
- ${D}/src/lib/MapOutputSpec.cc
- ${D}/src/lib/MemoryBlock.cc
- ${D}/src/lib/Merge.cc
- ${D}/src/lib/NativeLibrary.cc
- ${D}/src/lib/Iterator.cc
- ${D}/src/lib/NativeObjectFactory.cc
- ${D}/src/lib/NativeRuntimeJniImpl.cc
- ${D}/src/lib/NativeTask.cc
- ${D}/src/lib/SpillInfo.cc
- ${D}/src/lib/Path.cc
- ${D}/src/lib/Streams.cc
- ${D}/src/lib/TaskCounters.cc
- ${D}/src/util/Checksum.cc
- ${D}/src/util/Random.cc
- ${D}/src/util/StringUtil.cc
- ${D}/src/util/SyncUtils.cc
- ${D}/src/util/Timer.cc
- ${D}/src/util/WritableUtils.cc
+ ${SRC}/src/handler/BatchHandler.cc
+ ${SRC}/src/handler/MCollectorOutputHandler.cc
+ ${SRC}/src/handler/AbstractMapHandler.cc
+ ${SRC}/src/handler/CombineHandler.cc
+ ${SRC}/src/lib/Buffers.cc
+ ${SRC}/src/lib/BufferStream.cc
+ ${SRC}/src/lib/Compressions.cc
+ ${SRC}/src/lib/PartitionBucket.cc
+ ${SRC}/src/lib/PartitionBucketIterator.cc
+ ${SRC}/src/lib/FileSystem.cc
+ ${SRC}/src/lib/IFile.cc
+ ${SRC}/src/lib/jniutils.cc
+ ${SRC}/src/lib/Log.cc
+ ${SRC}/src/lib/MapOutputCollector.cc
+ ${SRC}/src/lib/MapOutputSpec.cc
+ ${SRC}/src/lib/MemoryBlock.cc
+ ${SRC}/src/lib/Merge.cc
+ ${SRC}/src/lib/NativeLibrary.cc
+ ${SRC}/src/lib/Iterator.cc
+ ${SRC}/src/lib/NativeObjectFactory.cc
+ ${SRC}/src/lib/NativeRuntimeJniImpl.cc
+ ${SRC}/src/lib/NativeTask.cc
+ ${SRC}/src/lib/SpillInfo.cc
+ ${SRC}/src/lib/Path.cc
+ ${SRC}/src/lib/Streams.cc
+ ${SRC}/src/lib/TaskCounters.cc
+ ${SRC}/src/util/Checksum.cc
+ ${SRC}/src/util/Random.cc
+ ${SRC}/src/util/StringUtil.cc
+ ${SRC}/src/util/SyncUtils.cc
+ ${SRC}/src/util/Timer.cc
+ ${SRC}/src/util/WritableUtils.cc
)
target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
-add_library(gtest ${D}/gtest/gtest-all.cc)
+add_library(gtest ${SRC}/gtest/gtest-all.cc)
set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
add_executable(nttest
- ${D}/test/lib/TestByteArray.cc
- ${D}/test/lib/TestByteBuffer.cc
- ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
- ${D}/test/lib/TestComparatorForStdSort.cc
- ${D}/test/lib/TestFixSizeContainer.cc
- ${D}/test/lib/TestMemoryPool.cc
- ${D}/test/lib/TestIterator.cc
- ${D}/test/lib/TestKVBuffer.cc
- ${D}/test/lib/TestMemBlockIterator.cc
- ${D}/test/lib/TestMemoryBlock.cc
- ${D}/test/lib/TestPartitionBucket.cc
- ${D}/test/lib/TestReadBuffer.cc
- ${D}/test/lib/TestReadWriteBuffer.cc
- ${D}/test/util/TestChecksum.cc
- ${D}/test/util/TestStringUtil.cc
- ${D}/test/util/TestWritableUtils.cc
- ${D}/test/TestCommand.cc
- ${D}/test/TestConfig.cc
- ${D}/test/TestCounter.cc
- ${D}/test/TestCompressions.cc
- ${D}/test/TestFileSystem.cc
- ${D}/test/TestIFile.cc
- ${D}/test/TestPrimitives.cc
- ${D}/test/TestSort.cc
- ${D}/test/TestMain.cc
- ${D}/test/test_commons.cc)
-
+ ${SRC}/test/lib/TestByteArray.cc
+ ${SRC}/test/lib/TestByteBuffer.cc
+ ${SRC}/test/lib/TestComparatorForDualPivotQuickSort.cc
+ ${SRC}/test/lib/TestComparatorForStdSort.cc
+ ${SRC}/test/lib/TestFixSizeContainer.cc
+ ${SRC}/test/lib/TestMemoryPool.cc
+ ${SRC}/test/lib/TestIterator.cc
+ ${SRC}/test/lib/TestKVBuffer.cc
+ ${SRC}/test/lib/TestMemBlockIterator.cc
+ ${SRC}/test/lib/TestMemoryBlock.cc
+ ${SRC}/test/lib/TestPartitionBucket.cc
+ ${SRC}/test/lib/TestReadBuffer.cc
+ ${SRC}/test/lib/TestReadWriteBuffer.cc
+ ${SRC}/test/util/TestChecksum.cc
+ ${SRC}/test/util/TestStringUtil.cc
+ ${SRC}/test/util/TestWritableUtils.cc
+ ${SRC}/test/TestCommand.cc
+ ${SRC}/test/TestConfig.cc
+ ${SRC}/test/TestCounter.cc
+ ${SRC}/test/TestCompressions.cc
+ ${SRC}/test/TestFileSystem.cc
+ ${SRC}/test/TestIFile.cc
+ ${SRC}/test/TestPrimitives.cc
+ ${SRC}/test/TestSort.cc
+ ${SRC}/test/TestMain.cc
+ ${SRC}/test/test_commons.cc)
target_link_libraries(nttest
nativetask_static
@@ -255,17 +189,14 @@ target_link_libraries(nttest
${NT_DEPEND_LIBRARY}
)
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- #
- # By embedding '$ORIGIN' into the RPATH of libnativetask.so,
- # dlopen will look in the directory containing libnativetask.so.
- # However, $ORIGIN is not supported by all operating systems.
- #
- SET_TARGET_PROPERTIES(nativetask
- PROPERTIES INSTALL_RPATH "\$ORIGIN/")
-ENDIF()
-
-SET(LIBNATIVETASK_VERSION "1.0.0")
-SET_TARGET_PROPERTIES(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
-dual_output_directory(nativetask target/usr/local/lib)
-output_directory(nttest test)
+# By embedding '$ORIGIN' into the RPATH of libnativetask.so, dlopen will look in
+# the directory containing libnativetask.so. However, $ORIGIN is not supported by
+# all operating systems.
+if(CMAKE_SYSTEM_NAME MATCHES "Linux|SunOS")
+ set_target_properties(nativetask PROPERTIES INSTALL_RPATH "\$ORIGIN/")
+endif()
+
+set(LIBNATIVETASK_VERSION "1.0.0")
+set_target_properties(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
+hadoop_dual_output_directory(nativetask target/usr/local/lib)
+hadoop_output_directory(nttest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a07c6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
deleted file mode 100644
index 0100b06..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-
-# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
-# This variable is set by maven.
-if (JVM_ARCH_DATA_MODEL EQUAL 32)
- # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64
- if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
- set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
- endif ()
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- # Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use
- # the 32-bit version of libjvm.so.
- set(CMAKE_SYSTEM_PROCESSOR "i686")
- endif ()
-endif (JVM_ARCH_DATA_MODEL EQUAL 32)
-
-# Determine float ABI of JVM on ARM Linux
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
- find_program(READELF readelf)
- if (READELF MATCHES "NOTFOUND")
- message(WARNING "readelf not found; JVM float ABI detection disabled")
- else (READELF MATCHES "NOTFOUND")
- execute_process(
- COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY}
- OUTPUT_VARIABLE JVM_ELF_ARCH
- ERROR_QUIET)
- if (NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers")
- message("Soft-float JVM detected")
-
- # Test compilation with -mfloat-abi=softfp using an arbitrary libc function
- # (typically fails with "fatal error: bits/predefs.h: No such file or directory"
- # if soft-float dev libraries are not installed)
- include(CMakePushCheckState)
- cmake_push_check_state()
- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -mfloat-abi=softfp")
- include(CheckSymbolExists)
- check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE)
- if (NOT SOFTFP_AVAILABLE)
- message(FATAL_ERROR "Soft-float dev libraries required (e.g. 'apt-get install libc6-dev-armel' on Debian/Ubuntu)")
- endif (NOT SOFTFP_AVAILABLE)
- cmake_pop_check_state()
-
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfloat-abi=softfp")
- endif ()
- endif (READELF MATCHES "NOTFOUND")
-endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
-
-IF("${CMAKE_SYSTEM}" MATCHES "Linux")
- #
- # Locate JNI_INCLUDE_DIRS and JNI_LIBRARIES.
- # Since we were invoked from Maven, we know that the JAVA_HOME environment
- # variable is valid. So we ignore system paths here and just use JAVA_HOME.
- #
- FILE(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME)
- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
- SET(_java_libarch "i386")
- ELSEIF (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- SET(_java_libarch "amd64")
- ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
- SET(_java_libarch "arm")
- ELSE()
- SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
- ENDIF()
- SET(_JDK_DIRS "${_JAVA_HOME}/jre/lib/${_java_libarch}/*"
- "${_JAVA_HOME}/jre/lib/${_java_libarch}"
- "${_JAVA_HOME}/jre/lib/*"
- "${_JAVA_HOME}/jre/lib"
- "${_JAVA_HOME}/lib/*"
- "${_JAVA_HOME}/lib"
- "${_JAVA_HOME}/include/*"
- "${_JAVA_HOME}/include"
- "${_JAVA_HOME}"
- )
- FIND_PATH(JAVA_INCLUDE_PATH
- NAMES jni.h
- PATHS ${_JDK_DIRS}
- NO_DEFAULT_PATH)
- #In IBM java, it's jniport.h instead of jni_md.h
- FIND_PATH(JAVA_INCLUDE_PATH2
- NAMES jni_md.h jniport.h
- PATHS ${_JDK_DIRS}
- NO_DEFAULT_PATH)
- SET(JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2})
- FIND_LIBRARY(JAVA_JVM_LIBRARY
- NAMES rt jvm
- PATHS ${_JDK_DIRS}
- NO_DEFAULT_PATH)
- SET(JNI_LIBRARIES ${JAVA_JVM_LIBRARY})
- MESSAGE("JAVA_HOME=${JAVA_HOME}, JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}")
- MESSAGE("JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}, JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}")
- IF(JAVA_JVM_LIBRARY AND JAVA_INCLUDE_PATH AND JAVA_INCLUDE_PATH2)
- MESSAGE("Located all JNI components successfully.")
- ELSE()
- MESSAGE(FATAL_ERROR "Failed to find a viable JVM installation under JAVA_HOME.")
- ENDIF()
-ELSE()
- find_package(JNI REQUIRED)
-ENDIF()
[30/48] hadoop git commit: MAPREDUCE-6418. MRApp should not shutdown
LogManager during shutdown. Contributed by Chang Li
Posted by zj...@apache.org.
MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7e62266
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7e62266
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7e62266
Branch: refs/heads/YARN-2928
Commit: d7e622663c19c7d5103d17e04a02cad07662cc7a
Parents: ef7b82b
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 18:01:22 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../apache/hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +++++++++++---
.../org/apache/hadoop/mapreduce/v2/app/MRApp.java | 15 +++++++++++++++
3 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7e62266/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index dd739ca..68ce49b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -551,6 +551,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be
logged in warn/info level (Chang Li via jlowe)
+ MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown
+ (Chang Li via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7e62266/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 149fd5d..f1d0b68 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1220,11 +1220,15 @@ public class MRAppMaster extends CompositeService {
startJobs();
}
}
-
+
+ protected void shutdownTaskLog() {
+ TaskLog.syncLogsShutdown(logSyncer);
+ }
+
@Override
public void stop() {
super.stop();
- TaskLog.syncLogsShutdown(logSyncer);
+ shutdownTaskLog();
}
private boolean isRecoverySupported() throws IOException {
@@ -1728,10 +1732,14 @@ public class MRAppMaster extends CompositeService {
T call(Configuration conf) throws Exception;
}
+ protected void shutdownLogManager() {
+ LogManager.shutdown();
+ }
+
@Override
protected void serviceStop() throws Exception {
super.serviceStop();
- LogManager.shutdown();
+ shutdownLogManager();
}
public ClientService getClientService() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7e62266/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
index 4fe4c44..b51adf2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
@@ -804,5 +804,20 @@ public class MRApp extends MRAppMaster {
new Text(containerToken.getService()));
return token.decodeIdentifier();
}
+
+ @Override
+ protected void shutdownTaskLog() {
+ // Avoid closing the logging system during unit tests,
+ // otherwise subsequent MRApp instances in the same test
+ // will fail to log anything.
+ }
+
+ @Override
+ protected void shutdownLogManager() {
+ // Avoid closing the logging system during unit tests,
+ // otherwise subsequent MRApp instances in the same test
+ // will fail to log anything.
+ }
+
}
[03/48] hadoop git commit: YARN-3695. ServerProxy (NMProxy,
etc.) shouldn't retry forever for non network exception. Contributed
by Raju Bairishetti
Posted by zj...@apache.org.
YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non network exception. Contributed by Raju Bairishetti
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f780d13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f780d13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f780d13
Branch: refs/heads/YARN-2928
Commit: 6f780d13ad9b8a71f80f6f8c5453eafc25bc1508
Parents: 38d1678
Author: Jian He <ji...@apache.org>
Authored: Mon Jun 29 13:37:32 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/ServerProxy.java | 21 ++---
.../containermanager/TestNMProxy.java | 81 +++++++++++++-------
3 files changed, 70 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f780d13/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0f4eef1..83b58ba 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -671,6 +671,9 @@ Release 2.8.0 - UNRELEASED
YARN-3860. rmadmin -transitionToActive should check the state of non-target node.
(Masatake Iwasaki via junping_du)
+ YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non
+ network exception. (Raju Bairishetti via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f780d13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
index e9bcf8d..de7fc7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
@@ -53,19 +53,22 @@ public class ServerProxy {
long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
long retryIntervalMS =
conf.getLong(connectRetryIntervalStr, defRetryInterval);
- if (maxWaitTime == -1) {
- // wait forever.
- return RetryPolicies.RETRY_FOREVER;
- }
- Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
- + maxWaitTimeStr + " should be a positive value.");
+ Preconditions.checkArgument((maxWaitTime == -1 || maxWaitTime > 0),
+ "Invalid Configuration. " + maxWaitTimeStr + " should be either"
+ + " positive value or -1.");
Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
+ connectRetryIntervalStr + "should be a positive value.");
- RetryPolicy retryPolicy =
- RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
- retryIntervalMS, TimeUnit.MILLISECONDS);
+ RetryPolicy retryPolicy = null;
+ if (maxWaitTime == -1) {
+ // wait forever.
+ retryPolicy = RetryPolicies.RETRY_FOREVER;
+ } else {
+ retryPolicy =
+ RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
+ retryIntervalMS, TimeUnit.MILLISECONDS);
+ }
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
new HashMap<Class<? extends Exception>, RetryPolicy>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f780d13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index 0b372be..102c9c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.io.retry.UnreliableInterface;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -58,8 +59,8 @@ public class TestNMProxy extends BaseContainerManagerTest {
@Before
public void setUp() throws Exception {
- conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
- conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
+ containerManager.start();
+ containerManager.setBlockNewContainerRequests(false);
}
@Override
@@ -77,7 +78,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
// This causes super to throw an NMNotYetReadyException
containerManager.setBlockNewContainerRequests(true);
} else {
- throw new java.net.ConnectException("start container exception");
+ if (isRetryPolicyRetryForEver()) {
+ // Throw non network exception
+ throw new IOException(
+ new UnreliableInterface.UnreliableException());
+ } else {
+ throw new java.net.ConnectException("start container exception");
+ }
}
} else {
// This stops super from throwing an NMNotYetReadyException
@@ -86,6 +93,11 @@ public class TestNMProxy extends BaseContainerManagerTest {
return super.startContainers(requests);
}
+ private boolean isRetryPolicyRetryForEver() {
+ return conf.getLong(
+ YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 1000) == -1;
+ }
+
@Override
public StopContainersResponse stopContainers(
StopContainersRequest requests) throws YarnException, IOException {
@@ -110,30 +122,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
}
@Test(timeout = 20000)
- public void testNMProxyRetry() throws Exception {
- containerManager.start();
- containerManager.setBlockNewContainerRequests(false);
- StartContainersRequest allRequests =
- Records.newRecord(StartContainersRequest.class);
- ApplicationId appId = ApplicationId.newInstance(1, 1);
- ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ public void testNMProxyRetry() throws Exception {
+ conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
+ conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
+ StartContainersRequest allRequests =
+ Records.newRecord(StartContainersRequest.class);
- org.apache.hadoop.yarn.api.records.Token nmToken =
- context.getNMTokenSecretManager().createNMToken(attemptId,
- context.getNodeId(), user);
- final InetSocketAddress address =
- conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
- YarnConfiguration.DEFAULT_NM_PORT);
- Token<NMTokenIdentifier> token =
- ConverterUtils.convertFromYarn(nmToken,
- SecurityUtil.buildTokenService(address));
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
- ugi.addToken(token);
-
- ContainerManagementProtocol proxy =
- NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
- YarnRPC.create(conf), address);
+ ContainerManagementProtocol proxy = getNMProxy();
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
@@ -156,4 +151,38 @@ public class TestNMProxy extends BaseContainerManagerTest {
proxy.startContainers(allRequests);
Assert.assertEquals(5, retryCount);
}
+
+ @Test(timeout = 20000, expected = IOException.class)
+ public void testShouldNotRetryForeverForNonNetworkExceptionsOnNMConnections()
+ throws Exception {
+ conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, -1);
+ StartContainersRequest allRequests =
+ Records.newRecord(StartContainersRequest.class);
+
+ ContainerManagementProtocol proxy = getNMProxy();
+
+ shouldThrowNMNotYetReadyException = false;
+ retryCount = 0;
+ proxy.startContainers(allRequests);
+ }
+
+ private ContainerManagementProtocol getNMProxy() {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+
+ org.apache.hadoop.yarn.api.records.Token nmToken =
+ context.getNMTokenSecretManager().createNMToken(attemptId,
+ context.getNodeId(), user);
+ final InetSocketAddress address =
+ conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_PORT);
+ Token<NMTokenIdentifier> token =
+ ConverterUtils.convertFromYarn(nmToken,
+ SecurityUtil.buildTokenService(address));
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ ugi.addToken(token);
+ return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
+ YarnRPC.create(conf), address);
+ }
}
[09/48] hadoop git commit: HDFS-8687. Remove the duplicate usage
message from Dfsck.java. (Contributed by Brahma Reddy Battula)
Posted by zj...@apache.org.
HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c51682b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c51682b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c51682b2
Branch: refs/heads/YARN-2928
Commit: c51682b2967211ddaf9e9cf4d0df70e16d1312e1
Parents: 94dca16
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jun 29 14:49:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:54 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51682b2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eb006eb..108a6c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -977,6 +977,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8628. Update missing command option for fetchdt
(J.Andreina via vinayakumarb)
+ HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Brahma
+ Reddy Battula via Arpit Agarwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51682b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 33033e7..6bb6603 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -377,7 +377,6 @@ public class DFSck extends Configured implements Tool {
int res = -1;
if ((args.length == 0) || ("-files".equals(args[0]))) {
printUsage(System.err);
- ToolRunner.printGenericCommandUsage(System.err);
} else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
res = 0;
} else {
[48/48] hadoop git commit: HDFS-8577. Avoid retrying to recover lease
on a file which does not exist (Contributed by J.Andreina)
Posted by zj...@apache.org.
HDFS-8577. Avoid retrying to recover lease on a file which does not exist (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06daeae6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06daeae6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06daeae6
Branch: refs/heads/YARN-2928
Commit: 06daeae65ad582600cc35174f52f728ad0f70573
Parents: 513eb8b
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Jul 3 13:35:48 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/tools/DebugAdmin.java | 10 ++++++++--
.../java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java | 8 ++++++++
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06daeae6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6678a3e..4f184fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1002,6 +1002,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html.
(Brahma Reddy Battula via Arpit Agarwal)
+ HDFS-8577. Avoid retrying to recover lease on a file which does not exist
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06daeae6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index 41f1ca4..d179a5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.tools;
import java.io.DataInputStream;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -254,6 +255,11 @@ public class DebugAdmin extends Configured implements Tool {
IOException ioe = null;
try {
recovered = dfs.recoverLease(new Path(pathStr));
+ } catch (FileNotFoundException e) {
+ System.err.println("recoverLease got exception: " + e.getMessage());
+ System.err.println("Giving up on recoverLease for " + pathStr +
+ " after 1 try");
+ return 1;
} catch (IOException e) {
ioe = e;
}
@@ -262,8 +268,8 @@ public class DebugAdmin extends Configured implements Tool {
return 0;
}
if (ioe != null) {
- System.err.println("recoverLease got exception: ");
- ioe.printStackTrace();
+ System.err.println("recoverLease got exception: " +
+ ioe.getMessage());
} else {
System.err.println("recoverLease returned false.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06daeae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
index 52b194d..07f70e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
@@ -37,6 +37,7 @@ import java.io.PrintStream;
import static org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil.*;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TestDebugAdmin {
private MiniDFSCluster cluster;
@@ -116,4 +117,11 @@ public class TestDebugAdmin {
"-block", blockFile.getAbsolutePath()})
);
}
+
+ @Test(timeout = 60000)
+ public void testRecoverLeaseforFileNotFound() throws Exception {
+ assertTrue(runCmd(new String[] {
+ "recoverLease", "-path", "/foo", "-retries", "2" }).contains(
+ "Giving up on recoverLease for /foo after 1 try"));
+ }
}
[06/48] hadoop git commit: HDFS-8653. Code cleanup for
DatanodeManager,
DatanodeDescriptor and DatanodeStorageInfo. Contributed by Zhe Zhang.
Posted by zj...@apache.org.
HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9d10052
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9d10052
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9d10052
Branch: refs/heads/YARN-2928
Commit: e9d100527b5a5b994d77770a5a6bf8a327d2139f
Parents: 130d7d1
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jun 29 12:12:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../CacheReplicationMonitor.java | 4 +--
.../blockmanagement/DatanodeDescriptor.java | 33 ++++++++++----------
.../server/blockmanagement/DatanodeManager.java | 26 +++++++--------
.../blockmanagement/DatanodeStorageInfo.java | 13 ++++++--
5 files changed, 45 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e55f340..0c56f2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -682,6 +682,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8623. Refactor NameNode handling of invalid, corrupt, and under-recovery
blocks. (Zhe Zhang via jing9)
+ HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and
+ DatanodeStorageInfo. (Zhe Zhang via wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index a0f3503..2f81ddf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -452,8 +452,8 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
file.getFullPathName(), cachedTotal, neededTotal);
}
- private String findReasonForNotCaching(CachedBlock cblock,
- BlockInfo blockInfo) {
+ private String findReasonForNotCaching(CachedBlock cblock,
+ BlockInfo blockInfo) {
if (blockInfo == null) {
// Somehow, a cache report with the block arrived, but the block
// reports from the DataNode haven't (yet?) described such a block.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index dd7b301..99def6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -64,7 +64,8 @@ public class DatanodeDescriptor extends DatanodeInfo {
// Stores status of decommissioning.
// If node is not decommissioning, do not use this object for anything.
- public final DecommissioningStatus decommissioningStatus = new DecommissioningStatus();
+ public final DecommissioningStatus decommissioningStatus =
+ new DecommissioningStatus();
private long curBlockReportId = 0;
@@ -115,7 +116,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
return null;
}
- List<E> results = new ArrayList<E>();
+ List<E> results = new ArrayList<>();
for(; !blockq.isEmpty() && numBlocks > 0; numBlocks--) {
results.add(blockq.poll());
}
@@ -135,7 +136,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
private final Map<String, DatanodeStorageInfo> storageMap =
- new HashMap<String, DatanodeStorageInfo>();
+ new HashMap<>();
/**
* A list of CachedBlock objects on this datanode.
@@ -217,12 +218,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
private long bandwidth;
/** A queue of blocks to be replicated by this datanode */
- private final BlockQueue<BlockTargetPair> replicateBlocks = new BlockQueue<BlockTargetPair>();
+ private final BlockQueue<BlockTargetPair> replicateBlocks =
+ new BlockQueue<>();
/** A queue of blocks to be recovered by this datanode */
private final BlockQueue<BlockInfoUnderConstruction> recoverBlocks =
- new BlockQueue<BlockInfoUnderConstruction>();
+ new BlockQueue<>();
/** A set of blocks to be invalidated by this datanode */
- private final LightWeightHashSet<Block> invalidateBlocks = new LightWeightHashSet<Block>();
+ private final LightWeightHashSet<Block> invalidateBlocks =
+ new LightWeightHashSet<>();
/* Variables for maintaining number of blocks scheduled to be written to
* this storage. This count is approximate and might be slightly bigger
@@ -230,9 +233,9 @@ public class DatanodeDescriptor extends DatanodeInfo {
* while writing the block).
*/
private EnumCounters<StorageType> currApproxBlocksScheduled
- = new EnumCounters<StorageType>(StorageType.class);
+ = new EnumCounters<>(StorageType.class);
private EnumCounters<StorageType> prevApproxBlocksScheduled
- = new EnumCounters<StorageType>(StorageType.class);
+ = new EnumCounters<>(StorageType.class);
private long lastBlocksScheduledRollTime = 0;
private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min
private int volumeFailures = 0;
@@ -276,6 +279,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
return storageMap.get(storageID);
}
}
+
DatanodeStorageInfo[] getStorageInfos() {
synchronized (storageMap) {
final Collection<DatanodeStorageInfo> storages = storageMap.values();
@@ -321,7 +325,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
Long.toHexString(curBlockReportId));
iter.remove();
if (zombies == null) {
- zombies = new LinkedList<DatanodeStorageInfo>();
+ zombies = new LinkedList<>();
}
zombies.add(storageInfo);
}
@@ -350,10 +354,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
*/
boolean removeBlock(String storageID, BlockInfo b) {
DatanodeStorageInfo s = getStorageInfo(storageID);
- if (s != null) {
- return s.removeBlock(b);
- }
- return false;
+ return s != null && s.removeBlock(b);
}
public void resetBlocks() {
@@ -449,7 +450,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
+ this.volumeFailures + " to " + volFailures);
synchronized (storageMap) {
failedStorageInfos =
- new HashSet<DatanodeStorageInfo>(storageMap.values());
+ new HashSet<>(storageMap.values());
}
}
@@ -505,7 +506,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
HashMap<String, DatanodeStorageInfo> excessStorages;
// Init excessStorages with all known storages.
- excessStorages = new HashMap<String, DatanodeStorageInfo>(storageMap);
+ excessStorages = new HashMap<>(storageMap);
// Remove storages that the DN reported in the heartbeat.
for (final StorageReport report : reports) {
@@ -542,7 +543,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
private final List<Iterator<BlockInfo>> iterators;
private BlockIterator(final DatanodeStorageInfo... storages) {
- List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
+ List<Iterator<BlockInfo>> iterators = new ArrayList<>();
for (DatanodeStorageInfo e : storages) {
iterators.add(e.getBlockIterator());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 8143fb4..4266004 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -85,7 +85,7 @@ public class DatanodeManager {
* Mapping: StorageID -> DatanodeDescriptor
*/
private final Map<String, DatanodeDescriptor> datanodeMap
- = new HashMap<String, DatanodeDescriptor>();
+ = new HashMap<>();
/** Cluster network topology */
private final NetworkTopology networktopology;
@@ -162,7 +162,7 @@ public class DatanodeManager {
* Software version -> Number of datanodes with this version
*/
private HashMap<String, Integer> datanodesSoftwareVersions =
- new HashMap<String, Integer>(4, 0.75f);
+ new HashMap<>(4, 0.75f);
/**
* The minimum time between resending caching directives to Datanodes,
@@ -217,7 +217,7 @@ public class DatanodeManager {
// locations of those hosts in the include list and store the mapping
// in the cache; so future calls to resolve will be fast.
if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
- final ArrayList<String> locations = new ArrayList<String>();
+ final ArrayList<String> locations = new ArrayList<>();
for (InetSocketAddress addr : hostFileManager.getIncludes()) {
locations.add(addr.getAddress().getHostAddress());
}
@@ -370,7 +370,7 @@ public class DatanodeManager {
// here we should get node but not datanode only .
Node client = getDatanodeByHost(targethost);
if (client == null) {
- List<String> hosts = new ArrayList<String> (1);
+ List<String> hosts = new ArrayList<> (1);
hosts.add(targethost);
List<String> resolvedHosts = dnsToSwitchMapping.resolve(hosts);
if (resolvedHosts != null && !resolvedHosts.isEmpty()) {
@@ -522,7 +522,7 @@ public class DatanodeManager {
void datanodeDump(final PrintWriter out) {
synchronized (datanodeMap) {
Map<String,DatanodeDescriptor> sortedDatanodeMap =
- new TreeMap<String,DatanodeDescriptor>(datanodeMap);
+ new TreeMap<>(datanodeMap);
out.println("Metasave: Number of datanodes: " + datanodeMap.size());
for (DatanodeDescriptor node : sortedDatanodeMap.values()) {
out.println(node.dumpDatanode());
@@ -660,7 +660,7 @@ public class DatanodeManager {
private void countSoftwareVersions() {
synchronized(datanodeMap) {
- HashMap<String, Integer> versionCount = new HashMap<String, Integer>();
+ HashMap<String, Integer> versionCount = new HashMap<>();
for(DatanodeDescriptor dn: datanodeMap.values()) {
// Check isAlive too because right after removeDatanode(),
// isDatanodeDead() is still true
@@ -677,7 +677,7 @@ public class DatanodeManager {
public HashMap<String, Integer> getDatanodesSoftwareVersions() {
synchronized(datanodeMap) {
- return new HashMap<String, Integer> (this.datanodesSoftwareVersions);
+ return new HashMap<> (this.datanodesSoftwareVersions);
}
}
@@ -710,7 +710,7 @@ public class DatanodeManager {
*/
private String resolveNetworkLocation (DatanodeID node)
throws UnresolvedTopologyException {
- List<String> names = new ArrayList<String>(1);
+ List<String> names = new ArrayList<>(1);
if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
names.add(node.getIpAddr());
} else {
@@ -1000,7 +1000,7 @@ public class DatanodeManager {
// If the network location is invalid, clear the cached mappings
// so that we have a chance to re-add this DataNode with the
// correct network location later.
- List<String> invalidNodeNames = new ArrayList<String>(3);
+ List<String> invalidNodeNames = new ArrayList<>(3);
// clear cache for nodes in IP or Hostname
invalidNodeNames.add(nodeReg.getIpAddr());
invalidNodeNames.add(nodeReg.getHostName());
@@ -1275,7 +1275,7 @@ public class DatanodeManager {
final HostFileManager.HostSet excludedNodes = hostFileManager.getExcludes();
synchronized(datanodeMap) {
- nodes = new ArrayList<DatanodeDescriptor>(datanodeMap.size());
+ nodes = new ArrayList<>(datanodeMap.size());
for (DatanodeDescriptor dn : datanodeMap.values()) {
final boolean isDead = isDatanodeDead(dn);
final boolean isDecommissioning = dn.isDecommissionInProgress();
@@ -1351,7 +1351,7 @@ public class DatanodeManager {
VolumeFailureSummary volumeFailureSummary) throws IOException {
synchronized (heartbeatManager) {
synchronized (datanodeMap) {
- DatanodeDescriptor nodeinfo = null;
+ DatanodeDescriptor nodeinfo;
try {
nodeinfo = getDatanode(nodeReg);
} catch(UnregisteredNodeException e) {
@@ -1389,7 +1389,7 @@ public class DatanodeManager {
final DatanodeStorageInfo[] storages = b.getExpectedStorageLocations();
// Skip stale nodes during recovery - not heart beated for some time (30s by default).
final List<DatanodeStorageInfo> recoveryLocations =
- new ArrayList<DatanodeStorageInfo>(storages.length);
+ new ArrayList<>(storages.length);
for (int i = 0; i < storages.length; i++) {
if (!storages[i].getDatanodeDescriptor().isStale(staleInterval)) {
recoveryLocations.add(storages[i]);
@@ -1431,7 +1431,7 @@ public class DatanodeManager {
return new DatanodeCommand[] { brCommand };
}
- final List<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
+ final List<DatanodeCommand> cmds = new ArrayList<>();
//check pending replication
List<BlockTargetPair> pendingList = nodeinfo.getReplicationCommand(
maxTransfers);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 65b83e1..92841a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -37,8 +37,9 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport;
public class DatanodeStorageInfo {
public static final DatanodeStorageInfo[] EMPTY_ARRAY = {};
- public static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) {
- return toDatanodeInfos(Arrays.asList(storages));
+ public static DatanodeInfo[] toDatanodeInfos(
+ DatanodeStorageInfo[] storages) {
+ return storages == null ? null: toDatanodeInfos(Arrays.asList(storages));
}
static DatanodeInfo[] toDatanodeInfos(List<DatanodeStorageInfo> storages) {
final DatanodeInfo[] datanodes = new DatanodeInfo[storages.size()];
@@ -58,6 +59,9 @@ public class DatanodeStorageInfo {
}
public static String[] toStorageIDs(DatanodeStorageInfo[] storages) {
+ if (storages == null) {
+ return null;
+ }
String[] storageIDs = new String[storages.length];
for(int i = 0; i < storageIDs.length; i++) {
storageIDs[i] = storages[i].getStorageID();
@@ -66,6 +70,9 @@ public class DatanodeStorageInfo {
}
public static StorageType[] toStorageTypes(DatanodeStorageInfo[] storages) {
+ if (storages == null) {
+ return null;
+ }
StorageType[] storageTypes = new StorageType[storages.length];
for(int i = 0; i < storageTypes.length; i++) {
storageTypes[i] = storages[i].getStorageType();
@@ -380,6 +387,6 @@ public class DatanodeStorageInfo {
}
static enum AddBlockResult {
- ADDED, REPLACED, ALREADY_EXIST;
+ ADDED, REPLACED, ALREADY_EXIST
}
}
[32/48] hadoop git commit: YARN-3830.
AbstractYarnScheduler.createReleaseCache may try to clean a null attempt.
Contributed by nijel.
Posted by zj...@apache.org.
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a
null attempt. Contributed by nijel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14bfd5d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14bfd5d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14bfd5d3
Branch: refs/heads/YARN-2928
Commit: 14bfd5d3c637f50d252fd44ef8396c8777272819
Parents: 09cd3df
Author: Devaraj K <de...@apache.org>
Authored: Wed Jul 1 19:03:44 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/AbstractYarnScheduler.java | 34 +++++----
.../scheduler/TestAbstractYarnScheduler.java | 74 ++++++++++++++++++--
3 files changed, 91 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bfd5d3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index abba1b8..fff46b8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -686,6 +686,9 @@ Release 2.8.0 - UNRELEASED
YARN-3823. Fix mismatch in default values for
yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
+ YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
+ attempt. (nijel via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bfd5d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ae927f1..aad76fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.SettableFuture;
@@ -451,25 +452,30 @@ public abstract class AbstractYarnScheduler
new Timer().schedule(new TimerTask() {
@Override
public void run() {
- for (SchedulerApplication<T> app : applications.values()) {
+ clearPendingContainerCache();
+ LOG.info("Release request cache is cleaned up");
+ }
+ }, nmExpireInterval);
+ }
- T attempt = app.getCurrentAppAttempt();
- synchronized (attempt) {
- for (ContainerId containerId : attempt.getPendingRelease()) {
- RMAuditLogger.logFailure(
- app.getUser(),
+ @VisibleForTesting
+ public void clearPendingContainerCache() {
+ for (SchedulerApplication<T> app : applications.values()) {
+ T attempt = app.getCurrentAppAttempt();
+ if (attempt != null) {
+ synchronized (attempt) {
+ for (ContainerId containerId : attempt.getPendingRelease()) {
+ RMAuditLogger.logFailure(app.getUser(),
AuditConstants.RELEASE_CONTAINER,
- "Unauthorized access or invalid container",
- "Scheduler",
- "Trying to release container not owned by app or with invalid id.",
- attempt.getApplicationId(), containerId);
- }
- attempt.getPendingRelease().clear();
+ "Unauthorized access or invalid container", "Scheduler",
+ "Trying to release container not owned by app "
+ + "or with invalid id.", attempt.getApplicationId(),
+ containerId);
}
+ attempt.getPendingRelease().clear();
}
- LOG.info("Release request cache is cleaned up");
}
- }, nmExpireInterval);
+ }
}
// clean up a completed container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bfd5d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 48ce822..91dd249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -18,26 +18,36 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.HashMap;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
@SuppressWarnings("unchecked")
public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
@@ -341,6 +351,58 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
}
}
+ /*
+ * This test case is to test the pending containers are cleared from the
+ * attempt even if one of the application in the list have current attempt as
+ * null (no attempt).
+ */
+ @SuppressWarnings({ "rawtypes" })
+ @Test(timeout = 10000)
+ public void testReleasedContainerIfAppAttemptisNull() throws Exception {
+ YarnConfiguration conf=getConf();
+ conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ MockRM rm1 = new MockRM(conf, memStore);
+ try {
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm1.getResourceScheduler();
+ // Mock App without attempt
+ RMApp mockAPp =
+ new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW);
+ SchedulerApplication<FiCaSchedulerApp> application =
+ new SchedulerApplication<FiCaSchedulerApp>(null, mockAPp.getUser());
+
+ // Second app with one app attempt
+ RMApp app = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1);
+ final ContainerId runningContainer =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ am1.allocate(null, Arrays.asList(runningContainer));
+
+ Map schedulerApplications = scheduler.getSchedulerApplications();
+ SchedulerApplication schedulerApp =
+ (SchedulerApplication) scheduler.getSchedulerApplications().get(
+ app.getApplicationId());
+ schedulerApplications.put(mockAPp.getApplicationId(), application);
+
+ scheduler.clearPendingContainerCache();
+
+ Assert.assertEquals("Pending containers are not released "
+ + "when one of the application attempt is null !", schedulerApp
+ .getCurrentAppAttempt().getPendingRelease().size(), 0);
+ } finally {
+ if (rm1 != null) {
+ rm1.stop();
+ }
+ }
+ }
+
private void verifyMaximumResourceCapability(
Resource expectedMaximumResource, AbstractYarnScheduler scheduler) {
[34/48] hadoop git commit: HDFS-8706. Fix typo in datanode startup
options in HDFSCommands.html. (Controibuted by Brahma Reddy Battula)
Posted by zj...@apache.org.
HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html. (Controibuted by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d57f65e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d57f65e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d57f65e6
Branch: refs/heads/YARN-2928
Commit: d57f65e6a18f2eef3c772196075c1e0e4d21b9a2
Parents: 23ac91c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jul 1 10:36:04 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57f65e6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 65f286d..3736932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -992,6 +992,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8579. Update HDFS usage with missing options
(J.Andreina via vinayakumarb)
+ HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html.
+ (Brahma Reddy Battula via Arpit Agarwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57f65e6/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 094dda1..1c2026c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -298,7 +298,7 @@ See the [HDFS Transparent Encryption Documentation](./TransparentEncryption.html
### `datanode`
-Usage: `hdfs datanode [-regular | -rollback | -rollingupgrace rollback]`
+Usage: `hdfs datanode [-regular | -rollback | -rollingupgrade rollback]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
[39/48] hadoop git commit: YARN-3875.
FSSchedulerNode#reserveResource() doesn't print Application Id properly in
log. Contributed by Bibin A Chundatt.
Posted by zj...@apache.org.
YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
properly in log. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec285b32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec285b32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec285b32
Branch: refs/heads/YARN-2928
Commit: ec285b3259c8a369b88ff44724d1670633a64cc6
Parents: a8a6574
Author: Devaraj K <de...@apache.org>
Authored: Thu Jul 2 10:20:31 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/scheduler/fair/FSSchedulerNode.java | 11 ++++++-----
2 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec285b32/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3e42e61..9780b4f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -689,6 +689,9 @@ Release 2.8.0 - UNRELEASED
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
attempt. (nijel via devaraj)
+ YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
+ properly in log. (Bibin A Chundatt via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec285b32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index be08dff..c86201a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -68,12 +68,13 @@ public class FSSchedulerNode extends SchedulerNode {
" on node " + this);
}
- LOG.info("Updated reserved container " +
- container.getContainer().getId() + " on node " +
- this + " for application " + application);
+ LOG.info("Updated reserved container " + container.getContainer().getId()
+ + " on node " + this + " for application "
+ + application.getApplicationId());
} else {
- LOG.info("Reserved container " + container.getContainer().getId() +
- " on node " + this + " for application " + application);
+ LOG.info("Reserved container " + container.getContainer().getId()
+ + " on node " + this + " for application "
+ + application.getApplicationId());
}
setReservedContainer(container);
this.reservedAppSchedulable = (FSAppAttempt) application;
[08/48] hadoop git commit: YARN-3770. SerializedException should also
handle java.lang.Error on de-serialization. Contributed by Lavkesh Lahngir
Posted by zj...@apache.org.
YARN-3770. SerializedException should also handle java.lang.Error on de-serialization. Contributed by Lavkesh Lahngir
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcb992af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcb992af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcb992af
Branch: refs/heads/YARN-2928
Commit: bcb992af7ae0d0317a3cf314c75090642bbab603
Parents: 3aff2fd
Author: Jian He <ji...@apache.org>
Authored: Mon Jun 29 14:31:32 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:02 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../api/records/impl/pb/SerializedExceptionPBImpl.java | 2 +-
.../records/impl/pb/TestSerializedExceptionPBImpl.java | 12 ++++++++++--
3 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcb992af/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 83b58ba..d8cfa4b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -674,6 +674,9 @@ Release 2.8.0 - UNRELEASED
YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non
network exception. (Raju Bairishetti via jianhe)
+ YARN-3770. SerializedException should also handle java.lang.Error on
+ de-serialization. (Lavkesh Lahngir via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcb992af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
index fd9e170..2ec232e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
@@ -101,7 +101,7 @@ public class SerializedExceptionPBImpl extends SerializedException {
} else if (RuntimeException.class.isAssignableFrom(realClass)) {
classType = RuntimeException.class;
} else {
- classType = Exception.class;
+ classType = Throwable.class;
}
return instantiateException(realClass.asSubclass(classType), getMessage(),
cause == null ? null : cause.deSerialize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcb992af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
index ac7e40e..ecfa63e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
@@ -20,10 +20,9 @@ package org.apache.hadoop.yarn.api.records.impl.pb;
import java.nio.channels.ClosedChannelException;
-import org.junit.Assert;
-import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
+import org.junit.Assert;
import org.junit.Test;
public class TestSerializedExceptionPBImpl {
@@ -79,4 +78,13 @@ public class TestSerializedExceptionPBImpl {
SerializedExceptionPBImpl pb3 = new SerializedExceptionPBImpl();
Assert.assertEquals(defaultProto.getTrace(), pb3.getRemoteTrace());
}
+
+ @Test
+ public void testThrowableDeserialization() {
+ // java.lang.Error should also be serializable
+ Error ex = new Error();
+ SerializedExceptionPBImpl pb = new SerializedExceptionPBImpl();
+ pb.init(ex);
+ Assert.assertEquals(ex.getClass(), pb.deSerialize().getClass());
+ }
}
[28/48] hadoop git commit: HADOOP-12116. Fix unrecommended syntax
usages in hadoop/hdfs/yarn script for cygwin in branch-2. Contributed by Li
Lu.
Posted by zj...@apache.org.
HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for cygwin in branch-2. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41bde4fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41bde4fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41bde4fc
Branch: refs/heads/YARN-2928
Commit: 41bde4fcbd5c25c18eeee48f435403006d8d0d2b
Parents: 36a33eb
Author: cnauroth <cn...@apache.org>
Authored: Tue Jun 30 16:54:30 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41bde4fc/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4e5f6bb..74d428f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -913,6 +913,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil
and fix for HA namespaces (rchiang via rkanter)
+ HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for
+ cygwin in branch-2. (Li Lu via cnauroth)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
[29/48] hadoop git commit: HADOOP-12164. Fix TestMove and
TestFsShellReturnCode failed to get command name using reflection. (Lei Xu)
Posted by zj...@apache.org.
HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command name using reflection. (Lei Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21765401
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21765401
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21765401
Branch: refs/heads/YARN-2928
Commit: 21765401e48c9eb578cd9b1f98ea7e4e385bc4fd
Parents: 41bde4f
Author: Lei Xu <le...@apache.org>
Authored: Tue Jun 30 17:26:49 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../apache/hadoop/fs/TestFsShellReturnCode.java | 60 +++++++++++++++-----
.../org/apache/hadoop/fs/shell/TestMove.java | 1 +
3 files changed, 49 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21765401/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 74d428f..39e2e5e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -916,6 +916,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for
cygwin in branch-2. (Li Lu via cnauroth)
+ HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
+ name using reflection. (Lei (Eddy) Xu)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21765401/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
index dcc19df..a2276ec 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -402,7 +403,7 @@ public class TestFsShellReturnCode {
// processing a file throws an interrupt, it should blow on first file
assertEquals(1, InterruptCommand.processed);
assertEquals(130, exitCode);
-
+
exitCode = shell.run(
new String[]{ "-testInterrupt", d.toString() });
// processing a file throws an interrupt, it should blow on file
@@ -412,17 +413,32 @@ public class TestFsShellReturnCode {
}
/**
+ * Faked Chown class for {@link testChownUserAndGroupValidity()}.
+ *
+ * The test only covers argument parsing, so override to skip processing.
+ */
+ private static class FakeChown extends FsShellPermissions.Chown {
+ public static String NAME = "chown";
+ @Override
+ protected void processArgument(PathData item) {
+ }
+ }
+
+ /**
* Tests combinations of valid and invalid user and group arguments to chown.
*/
@Test
public void testChownUserAndGroupValidity() {
- // This test only covers argument parsing, so override to skip processing.
- FsCommand chown = new FsShellPermissions.Chown() {
- @Override
- protected void processArgument(PathData item) {
- }
- };
- chown.setConf(new Configuration());
+ testChownUserAndGroupValidity(true);
+ testChownUserAndGroupValidity(false);
+ }
+
+ private void testChownUserAndGroupValidity(boolean enableWarning) {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
+ FsCommand chown = new FakeChown();
+ chown.setConf(conf);
// The following are valid (no exception expected).
chown.run("user", "/path");
@@ -447,17 +463,31 @@ public class TestFsShellReturnCode {
}
/**
+ * Faked Chgrp class for {@link testChgrpGroupValidity()}.
+ * The test only covers argument parsing, so override to skip processing.
+ */
+ private static class FakeChgrp extends FsShellPermissions.Chgrp {
+ public static String NAME = "chgrp";
+ @Override
+ protected void processArgument(PathData item) {
+ }
+ }
+
+ /**
* Tests valid and invalid group arguments to chgrp.
*/
@Test
public void testChgrpGroupValidity() {
- // This test only covers argument parsing, so override to skip processing.
- FsCommand chgrp = new FsShellPermissions.Chgrp() {
- @Override
- protected void processArgument(PathData item) {
- }
- };
- chgrp.setConf(new Configuration());
+ testChgrpGroupValidity(true);
+ testChgrpGroupValidity(false);
+ }
+
+ private void testChgrpGroupValidity(boolean enableWarning) {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
+ FsShellPermissions.Chgrp chgrp = new FakeChgrp();
+ chgrp.setConf(conf);
// The following are valid (no exception expected).
chgrp.run("group", "/path");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21765401/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
index 6599edf..94930e5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
@@ -114,6 +114,7 @@ public class TestMove {
}
private static class InstrumentedRenameCommand extends MoveCommands.Rename {
+ public static String NAME = "InstrumentedRename";
private Exception error = null;
@Override
public void displayError(Exception e) {
[07/48] hadoop git commit: HADOOP-12107. long running apps may have a
huge number of StatisticsData instances under FileSystem (Sangjin Lee via
Ming Ma)
Posted by zj...@apache.org.
HADOOP-12107. long running apps may have a huge number of StatisticsData instances under FileSystem (Sangjin Lee via Ming Ma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/296d1507
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/296d1507
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/296d1507
Branch: refs/heads/YARN-2928
Commit: 296d1507bea76acf37537e408f295c6f66c4f7d7
Parents: bcb992a
Author: Ming Ma <mi...@apache.org>
Authored: Mon Jun 29 14:37:38 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:02 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/FileSystem.java | 140 +++++++++++++------
.../apache/hadoop/fs/FCStatisticsBaseTest.java | 56 +++++++-
3 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/296d1507/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a9b44e3..50192ae 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -490,6 +490,9 @@ Trunk (Unreleased)
HADOOP-11347. RawLocalFileSystem#mkdir and create should honor umask (Varun
Saxena via Colin P. McCabe)
+ HADOOP-12107. long running apps may have a huge number of StatisticsData
+ instances under FileSystem (Sangjin Lee via Ming Ma)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/296d1507/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 3f9e3bd..1d7bc87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.fs;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.ref.WeakReference;
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
@@ -32,7 +33,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -2920,16 +2920,6 @@ public abstract class FileSystem extends Configured implements Closeable {
volatile int readOps;
volatile int largeReadOps;
volatile int writeOps;
- /**
- * Stores a weak reference to the thread owning this StatisticsData.
- * This allows us to remove StatisticsData objects that pertain to
- * threads that no longer exist.
- */
- final WeakReference<Thread> owner;
-
- StatisticsData(WeakReference<Thread> owner) {
- this.owner = owner;
- }
/**
* Add another StatisticsData object to this one.
@@ -3000,17 +2990,37 @@ public abstract class FileSystem extends Configured implements Closeable {
* Thread-local data.
*/
private final ThreadLocal<StatisticsData> threadData;
-
+
/**
- * List of all thread-local data areas. Protected by the Statistics lock.
+ * Set of all thread-local data areas. Protected by the Statistics lock.
+ * The references to the statistics data are kept using phantom references
+ * to the associated threads. Proper clean-up is performed by the cleaner
+ * thread when the threads are garbage collected.
*/
- private LinkedList<StatisticsData> allData;
+ private final Set<StatisticsDataReference> allData;
+
+ /**
+ * Global reference queue and a cleaner thread that manage statistics data
+ * references from all filesystem instances.
+ */
+ private static final ReferenceQueue<Thread> STATS_DATA_REF_QUEUE;
+ private static final Thread STATS_DATA_CLEANER;
+
+ static {
+ STATS_DATA_REF_QUEUE = new ReferenceQueue<Thread>();
+ // start a single daemon cleaner thread
+ STATS_DATA_CLEANER = new Thread(new StatisticsDataReferenceCleaner());
+ STATS_DATA_CLEANER.
+ setName(StatisticsDataReferenceCleaner.class.getName());
+ STATS_DATA_CLEANER.setDaemon(true);
+ STATS_DATA_CLEANER.start();
+ }
public Statistics(String scheme) {
this.scheme = scheme;
- this.rootData = new StatisticsData(null);
+ this.rootData = new StatisticsData();
this.threadData = new ThreadLocal<StatisticsData>();
- this.allData = null;
+ this.allData = new HashSet<StatisticsDataReference>();
}
/**
@@ -3020,7 +3030,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public Statistics(Statistics other) {
this.scheme = other.scheme;
- this.rootData = new StatisticsData(null);
+ this.rootData = new StatisticsData();
other.visitAll(new StatisticsAggregator<Void>() {
@Override
public void accept(StatisticsData data) {
@@ -3032,6 +3042,63 @@ public abstract class FileSystem extends Configured implements Closeable {
}
});
this.threadData = new ThreadLocal<StatisticsData>();
+ this.allData = new HashSet<StatisticsDataReference>();
+ }
+
+ /**
+ * A phantom reference to a thread that also includes the data associated
+ * with that thread. On the thread being garbage collected, it is enqueued
+ * to the reference queue for clean-up.
+ */
+ private class StatisticsDataReference extends PhantomReference<Thread> {
+ private final StatisticsData data;
+
+ public StatisticsDataReference(StatisticsData data, Thread thread) {
+ super(thread, STATS_DATA_REF_QUEUE);
+ this.data = data;
+ }
+
+ public StatisticsData getData() {
+ return data;
+ }
+
+ /**
+ * Performs clean-up action when the associated thread is garbage
+ * collected.
+ */
+ public void cleanUp() {
+ // use the statistics lock for safety
+ synchronized (Statistics.this) {
+ /*
+ * If the thread that created this thread-local data no longer exists,
+ * remove the StatisticsData from our list and fold the values into
+ * rootData.
+ */
+ rootData.add(data);
+ allData.remove(this);
+ }
+ }
+ }
+
+ /**
+ * Background action to act on references being removed.
+ */
+ private static class StatisticsDataReferenceCleaner implements Runnable {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ StatisticsDataReference ref =
+ (StatisticsDataReference)STATS_DATA_REF_QUEUE.remove();
+ ref.cleanUp();
+ } catch (Throwable th) {
+ // the cleaner thread should continue to run even if there are
+ // exceptions, including InterruptedException
+ LOG.warn("exception in the cleaner thread but it will continue to "
+ + "run", th);
+ }
+ }
+ }
}
/**
@@ -3040,14 +3107,12 @@ public abstract class FileSystem extends Configured implements Closeable {
public StatisticsData getThreadStatistics() {
StatisticsData data = threadData.get();
if (data == null) {
- data = new StatisticsData(
- new WeakReference<Thread>(Thread.currentThread()));
+ data = new StatisticsData();
threadData.set(data);
+ StatisticsDataReference ref =
+ new StatisticsDataReference(data, Thread.currentThread());
synchronized(this) {
- if (allData == null) {
- allData = new LinkedList<StatisticsData>();
- }
- allData.add(data);
+ allData.add(ref);
}
}
return data;
@@ -3105,21 +3170,9 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
visitor.accept(rootData);
- if (allData != null) {
- for (Iterator<StatisticsData> iter = allData.iterator();
- iter.hasNext(); ) {
- StatisticsData data = iter.next();
- visitor.accept(data);
- if (data.owner.get() == null) {
- /*
- * If the thread that created this thread-local data no
- * longer exists, remove the StatisticsData from our list
- * and fold the values into rootData.
- */
- rootData.add(data);
- iter.remove();
- }
- }
+ for (StatisticsDataReference ref: allData) {
+ StatisticsData data = ref.getData();
+ visitor.accept(data);
}
return visitor.aggregate();
}
@@ -3226,7 +3279,7 @@ public abstract class FileSystem extends Configured implements Closeable {
@Override
public String toString() {
return visitAll(new StatisticsAggregator<String>() {
- private StatisticsData total = new StatisticsData(null);
+ private StatisticsData total = new StatisticsData();
@Override
public void accept(StatisticsData data) {
@@ -3259,7 +3312,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public void reset() {
visitAll(new StatisticsAggregator<Void>() {
- private StatisticsData total = new StatisticsData(null);
+ private StatisticsData total = new StatisticsData();
@Override
public void accept(StatisticsData data) {
@@ -3281,6 +3334,11 @@ public abstract class FileSystem extends Configured implements Closeable {
public String getScheme() {
return scheme;
}
+
+ @VisibleForTesting
+ synchronized int getAllThreadLocalDataSize() {
+ return allData.size();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/296d1507/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 90337a6..3e33362 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -18,26 +18,34 @@
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.FileContextTestHelper.createFile;
+
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.base.Supplier;
import com.google.common.util.concurrent.Uninterruptibles;
-import static org.apache.hadoop.fs.FileContextTestHelper.*;
-
/**
* <p>
* Base class to test {@link FileContext} Statistics.
* </p>
*/
public abstract class FCStatisticsBaseTest {
-
static protected int blockSize = 512;
static protected int numBlocks = 1;
@@ -102,6 +110,48 @@ public abstract class FCStatisticsBaseTest {
fc.delete(filePath, true);
}
+ @Test(timeout=60000)
+ public void testStatisticsThreadLocalDataCleanUp() throws Exception {
+ final Statistics stats = new Statistics("test");
+ // create a small thread pool to test the statistics
+ final int size = 2;
+ ExecutorService es = Executors.newFixedThreadPool(size);
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(size);
+ for (int i = 0; i < size; i++) {
+ tasks.add(new Callable<Boolean>() {
+ public Boolean call() {
+ // this populates the data set in statistics
+ stats.incrementReadOps(1);
+ return true;
+ }
+ });
+ }
+ // run the threads
+ es.invokeAll(tasks);
+ // assert that the data size is exactly the number of threads
+ final AtomicInteger allDataSize = new AtomicInteger(0);
+ allDataSize.set(stats.getAllThreadLocalDataSize());
+ Assert.assertEquals(size, allDataSize.get());
+ Assert.assertEquals(size, stats.getReadOps());
+ // force the GC to collect the threads by shutting down the thread pool
+ es.shutdownNow();
+ es.awaitTermination(1, TimeUnit.MINUTES);
+ es = null;
+ System.gc();
+
+ // wait for up to 10 seconds
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ int size = stats.getAllThreadLocalDataSize();
+ allDataSize.set(size);
+ return size == 0;
+ }
+ }, 1000, 10*1000);
+ Assert.assertEquals(0, allDataSize.get());
+ Assert.assertEquals(size, stats.getReadOps());
+ }
+
/**
* Bytes read may be different for different file systems. This method should
* throw assertion error if bytes read are incorrect.
[22/48] hadoop git commit: HADOOP-12124. Add HTrace support for
FsShell (cmccabe)
Posted by zj...@apache.org.
HADOOP-12124. Add HTrace support for FsShell (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/349b9f98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/349b9f98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/349b9f98
Branch: refs/heads/YARN-2928
Commit: 349b9f984d0d3edf762491eb10cd54c1913ce2fd
Parents: 52d4f05
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:46:25 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/main/java/org/apache/hadoop/fs/FsShell.java | 16 ++++++++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/349b9f98/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4d69a229..19ba620 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -671,6 +671,8 @@ Release 2.8.0 - UNRELEASED
TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not
installed. (wang)
+ HADOOP-12124. Add HTrace support for FsShell (cmccabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/349b9f98/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index db73f6d..24b6339 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -33,8 +33,13 @@ import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.FsCommand;
import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.tracing.TraceUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.SamplerBuilder;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
/** Provide command line access to a FileSystem. */
@InterfaceAudience.Private
@@ -47,6 +52,7 @@ public class FsShell extends Configured implements Tool {
private FileSystem fs;
private Trash trash;
protected CommandFactory commandFactory;
+ private Sampler traceSampler;
private final String usagePrefix =
"Usage: hadoop fs [generic options]";
@@ -272,7 +278,8 @@ public class FsShell extends Configured implements Tool {
public int run(String argv[]) throws Exception {
// initialize FsShell
init();
-
+ traceSampler = new SamplerBuilder(TraceUtils.
+ wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
int exitCode = -1;
if (argv.length < 1) {
printUsage(System.err);
@@ -284,7 +291,12 @@ public class FsShell extends Configured implements Tool {
if (instance == null) {
throw new UnknownCommandException();
}
- exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));
+ TraceScope scope = Trace.startSpan(instance.getCommandName(), traceSampler);
+ try {
+ exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));
+ } finally {
+ scope.close();
+ }
} catch (IllegalArgumentException e) {
displayError(cmd, e.getLocalizedMessage());
if (instance != null) {
[26/48] hadoop git commit: YARN-3823. Fix mismatch in default values
for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray
Chiang.
Posted by zj...@apache.org.
YARN-3823. Fix mismatch in default values for
yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray
Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09cd3dfc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09cd3dfc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09cd3dfc
Branch: refs/heads/YARN-2928
Commit: 09cd3dfcdc3bdd0dd7f3f2223c30768a3f273ed9
Parents: 2176540
Author: Devaraj K <de...@apache.org>
Authored: Wed Jul 1 11:11:26 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09cd3dfc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5269bec..abba1b8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.8.0 - UNRELEASED
YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables.
(Zhihai Xu via gera)
+ YARN-3823. Fix mismatch in default values for
+ yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09cd3dfc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d34cf82..62527b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -276,7 +276,7 @@
in terms of virtual CPU cores. Requests higher than this will throw a
InvalidResourceRequestException.</description>
<name>yarn.scheduler.maximum-allocation-vcores</name>
- <value>32</value>
+ <value>4</value>
</property>
<property>
[16/48] hadoop git commit: YARN-3768. ArrayIndexOutOfBoundsException
with empty environment variables. (Zhihai Xu via gera)
Posted by zj...@apache.org.
YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables. (Zhihai Xu via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcac9c96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcac9c96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcac9c96
Branch: refs/heads/YARN-2928
Commit: bcac9c96b4e228239d364f583849cf6a788ffc6a
Parents: 9465dc4
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Jun 30 14:44:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/util/Shell.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../java/org/apache/hadoop/yarn/util/Apps.java | 19 ++++--
.../org/apache/hadoop/yarn/util/TestApps.java | 61 ++++++++++++++++++++
4 files changed, 81 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 45c1588..ed83e8d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -221,10 +221,12 @@ abstract public class Shell {
new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
}
+ public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
/** Return a regular expression string that match environment variables */
public static String getEnvironmentVariableRegex() {
- return (WINDOWS) ? "%([A-Za-z_][A-Za-z0-9_]*?)%" :
- "\\$([A-Za-z_][A-Za-z0-9_]*)";
+ return (WINDOWS)
+ ? "%(" + ENV_NAME_REGEX + "?)%"
+ : "\\$(" + ENV_NAME_REGEX + ")";
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d8cfa4b..896f54e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -677,6 +677,9 @@ Release 2.8.0 - UNRELEASED
YARN-3770. SerializedException should also handle java.lang.Error on
de-serialization. (Lavkesh Lahngir via jianhe)
+ YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables.
+ (Zhihai Xu via gera)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index cf3940f..9235e7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -44,6 +44,14 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
public class Apps {
public static final String APP = "application";
public static final String ID = "ID";
+ private static final Pattern VAR_SUBBER =
+ Pattern.compile(Shell.getEnvironmentVariableRegex());
+ private static final Pattern VARVAL_SPLITTER = Pattern.compile(
+ "(?<=^|,)" // preceded by ',' or line begin
+ + '(' + Shell.ENV_NAME_REGEX + ')' // var group
+ + '='
+ + "([^,]*)" // val group
+ );
public static ApplicationId toAppID(String aid) {
Iterator<String> it = _split(aid).iterator();
@@ -73,11 +81,10 @@ public class Apps {
public static void setEnvFromInputString(Map<String, String> env,
String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
- String childEnvs[] = envString.split(",");
- Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
- for (String cEnv : childEnvs) {
- String[] parts = cEnv.split("="); // split on '='
- Matcher m = p.matcher(parts[1]);
+ Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+ while (varValMatcher.find()) {
+ String envVar = varValMatcher.group(1);
+ Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
StringBuffer sb = new StringBuffer();
while (m.find()) {
String var = m.group(1);
@@ -93,7 +100,7 @@ public class Apps {
m.appendReplacement(sb, Matcher.quoteReplacement(replace));
}
m.appendTail(sb);
- addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+ addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
new file mode 100644
index 0000000..66d2d23
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.util;
+
+import org.apache.hadoop.util.Shell;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestApps {
+ @Test
+ public void testSetEnvFromInputString() {
+ Map<String, String> environment = new HashMap<String, String>();
+ environment.put("JAVA_HOME", "/path/jdk");
+ String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
+ + ",g_nix=$JAVA_HOME";
+ Apps.setEnvFromInputString(environment, goodEnv, File.pathSeparator);
+ assertEquals("1", environment.get("a1"));
+ assertEquals("2", environment.get("b_2"));
+ assertEquals("3", environment.get("_c"));
+ assertEquals("4", environment.get("d"));
+ assertEquals("", environment.get("e"));
+ if (Shell.WINDOWS) {
+ assertEquals("$JAVA_HOME", environment.get("g_nix"));
+ assertEquals("/path/jdk", environment.get("f_win"));
+ } else {
+ assertEquals("/path/jdk", environment.get("g_nix"));
+ assertEquals("%JAVA_HOME%", environment.get("f_win"));
+ }
+ String badEnv = "1,,2=a=b,3=a=,4==,5==a,==,c-3=3,=";
+ environment.clear();
+ Apps.setEnvFromInputString(environment, badEnv, File.pathSeparator);
+ assertEquals(environment.size(), 0);
+
+ // Test "=" in the value part
+ environment.clear();
+ Apps.setEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2",
+ File.pathSeparator);
+ assertEquals("=", environment.get("e1"));
+ assertEquals("a1=a2", environment.get("e2"));
+ }
+}
[35/48] hadoop git commit: YARN-3793. Several NPEs when deleting
local files on NM recovery. Contributed by Varun Saxena
Posted by zj...@apache.org.
YARN-3793. Several NPEs when deleting local files on NM recovery. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e556ee03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e556ee03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e556ee03
Branch: refs/heads/YARN-2928
Commit: e556ee030ff3610c8c407caa18eddd2a6c443d9d
Parents: d7e6226
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 21:13:32 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../logaggregation/AppLogAggregatorImpl.java | 6 ++--
.../TestLogAggregationService.java | 36 ++++++++++++++++++++
3 files changed, 42 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fff46b8..c4c7fa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -701,6 +701,9 @@ Release 2.7.2 - UNRELEASED
BUG FIXES
+ YARN-3793. Several NPEs when deleting local files on NM recovery (Varun
+ Saxena via jlowe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 4b95a03..654eb0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -276,10 +276,10 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
aggregator.doContainerLogAggregation(writer, appFinished);
if (uploadedFilePathsInThisCycle.size() > 0) {
uploadedLogsInThisCycle = true;
+ this.delService.delete(this.userUgi.getShortUserName(), null,
+ uploadedFilePathsInThisCycle
+ .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
}
- this.delService.delete(this.userUgi.getShortUserName(), null,
- uploadedFilePathsInThisCycle
- .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
// This container is finished, and all its logs have been uploaded,
// remove it from containerLogAggregators.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index fd97cef..6a3d270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -31,6 +31,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -280,6 +281,41 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
verifyLocalFileDeletion(logAggregationService);
}
+ /* Test to verify fix for YARN-3793 */
+ @Test
+ public void testNoLogsUploadedOnAppFinish() throws Exception {
+ this.delSrvc = new DeletionService(createContainerExecutor());
+ delSrvc = spy(delSrvc);
+ this.delSrvc.init(conf);
+ this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ this.remoteRootLogDir.getAbsolutePath());
+
+ LogAggregationService logAggregationService = new LogAggregationService(
+ dispatcher, this.context, this.delSrvc, super.dirsHandler);
+ logAggregationService.init(this.conf);
+ logAggregationService.start();
+
+ ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
+ File appLogDir = new File(localLogDir, ConverterUtils.toString(app));
+ appLogDir.mkdir();
+ LogAggregationContext context =
+ LogAggregationContext.newInstance("HOST*", "sys*");
+ logAggregationService.handle(new LogHandlerAppStartedEvent(app, this.user,
+ null, ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls, context));
+
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(app, 1);
+ ContainerId cont = BuilderUtils.newContainerId(appAttemptId, 1);
+ writeContainerLogs(appLogDir, cont, new String[] { "stdout",
+ "stderr", "syslog" });
+ logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont, 0));
+ logAggregationService.handle(new LogHandlerAppFinishedEvent(app));
+ logAggregationService.stop();
+ delSrvc.stop();
+ // Aggregated logs should not be deleted if not uploaded.
+ verify(delSrvc, times(0)).delete(user, null);
+ }
@Test
public void testNoContainerOnNode() throws Exception {
[44/48] hadoop git commit: HDFS-8686. WebHdfsFileSystem#getXAttr(Path
p,
final String name) doesn't work if namespace is in capitals (Contributed by
kanaka kumar avvaru)
Posted by zj...@apache.org.
HDFS-8686. WebHdfsFileSystem#getXAttr(Path p, final String name) doesn't work if namespace is in capitals (Contributed by kanaka kumar avvaru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68375526
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68375526
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68375526
Branch: refs/heads/YARN-2928
Commit: 68375526908332b9a5ec12228937b22dc18e82a2
Parents: 7182a03
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jul 6 16:09:24 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/web/JsonUtilClient.java | 14 ++++++++++++++
.../org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSXAttrBaseTest.java | 5 ++++-
.../TestOfflineImageViewerForXAttr.java | 3 +++
5 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index e025e31..713836c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -413,6 +413,20 @@ class JsonUtilClient {
return null;
}
+ /** Expecting only single XAttr in the map. return its value */
+ static byte[] getXAttr(final Map<?, ?> json) throws IOException {
+ if (json == null) {
+ return null;
+ }
+
+ Map<String, byte[]> xAttrs = toXAttrs(json);
+ if (xAttrs != null && !xAttrs.values().isEmpty()) {
+ return xAttrs.values().iterator().next();
+ }
+
+ return null;
+ }
+
static Map<String, byte[]> toXAttrs(final Map<?, ?> json)
throws IOException {
if (json == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 2650dca..b661d07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -963,7 +963,7 @@ public class WebHdfsFileSystem extends FileSystem
new XAttrEncodingParam(XAttrCodec.HEX)) {
@Override
byte[] decodeResponse(Map<?, ?> json) throws IOException {
- return JsonUtilClient.getXAttr(json, name);
+ return JsonUtilClient.getXAttr(json);
}
}.run();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4f184fb..9edc2af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1005,6 +1005,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8577. Avoid retrying to recover lease on a file which does not exist
(J.Andreina via vinayakumarb)
+ HDFS-8686. WebHdfsFileSystem#getXAttr(Path p, final String name) doesn't
+ work if namespace is in capitals (kanaka kumar avvaru via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
index e21e34c..eb9053c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
@@ -395,7 +395,10 @@ public class FSXAttrBaseTest {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE));
fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE));
-
+
+ final byte[] theValue = fs.getXAttr(path, "USER.a2");
+ Assert.assertArrayEquals(value2, theValue);
+
/* An XAttr that was requested does not exist. */
try {
final byte[] value = fs.getXAttr(path, name3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
index 3f23f64..6c82101 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
@@ -231,6 +231,9 @@ public class TestOfflineImageViewerForXAttr {
"user.attr1"));
assertEquals("value1", value);
+ value = new String(webhdfs.getXAttr(new Path("/dir1"), "USER.attr1"));
+ assertEquals("value1", value);
+
Map<String, byte[]> contentMap = webhdfs.getXAttrs(new Path("/dir1"),
names);
[47/48] hadoop git commit: YARN-3882. AggregatedLogFormat should
close aclScanner and ownerScanner after create them. Contributed by zhihai xu
Posted by zj...@apache.org.
YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner
after create them. Contributed by zhihai xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccfc7874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccfc7874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccfc7874
Branch: refs/heads/YARN-2928
Commit: ccfc78748569b49ffa34dcb64032222174489058
Parents: 06daeae
Author: Xuan <xg...@apache.org>
Authored: Sat Jul 4 21:51:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../logaggregation/AggregatedLogFormat.java | 83 +++++++++++---------
2 files changed, 49 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccfc7874/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9780b4f..91358c1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -692,6 +692,9 @@ Release 2.8.0 - UNRELEASED
YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
properly in log. (Bibin A Chundatt via devaraj)
+ YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner
+ after create them. (zhihai xu via xgong)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccfc7874/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index debe770..c9453b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -489,18 +489,23 @@ public class AggregatedLogFormat {
* @throws IOException
*/
public String getApplicationOwner() throws IOException {
- TFile.Reader.Scanner ownerScanner = reader.createScanner();
- LogKey key = new LogKey();
- while (!ownerScanner.atEnd()) {
- TFile.Reader.Scanner.Entry entry = ownerScanner.entry();
- key.readFields(entry.getKeyStream());
- if (key.toString().equals(APPLICATION_OWNER_KEY.toString())) {
- DataInputStream valueStream = entry.getValueStream();
- return valueStream.readUTF();
+ TFile.Reader.Scanner ownerScanner = null;
+ try {
+ ownerScanner = reader.createScanner();
+ LogKey key = new LogKey();
+ while (!ownerScanner.atEnd()) {
+ TFile.Reader.Scanner.Entry entry = ownerScanner.entry();
+ key.readFields(entry.getKeyStream());
+ if (key.toString().equals(APPLICATION_OWNER_KEY.toString())) {
+ DataInputStream valueStream = entry.getValueStream();
+ return valueStream.readUTF();
+ }
+ ownerScanner.advance();
}
- ownerScanner.advance();
+ return null;
+ } finally {
+ IOUtils.cleanup(LOG, ownerScanner);
}
- return null;
}
/**
@@ -513,38 +518,42 @@ public class AggregatedLogFormat {
public Map<ApplicationAccessType, String> getApplicationAcls()
throws IOException {
// TODO Seek directly to the key once a comparator is specified.
- TFile.Reader.Scanner aclScanner = reader.createScanner();
- LogKey key = new LogKey();
- Map<ApplicationAccessType, String> acls =
- new HashMap<ApplicationAccessType, String>();
- while (!aclScanner.atEnd()) {
- TFile.Reader.Scanner.Entry entry = aclScanner.entry();
- key.readFields(entry.getKeyStream());
- if (key.toString().equals(APPLICATION_ACL_KEY.toString())) {
- DataInputStream valueStream = entry.getValueStream();
- while (true) {
- String appAccessOp = null;
- String aclString = null;
- try {
- appAccessOp = valueStream.readUTF();
- } catch (EOFException e) {
- // Valid end of stream.
- break;
- }
- try {
- aclString = valueStream.readUTF();
- } catch (EOFException e) {
- throw new YarnRuntimeException("Error reading ACLs", e);
+ TFile.Reader.Scanner aclScanner = null;
+ try {
+ aclScanner = reader.createScanner();
+ LogKey key = new LogKey();
+ Map<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>();
+ while (!aclScanner.atEnd()) {
+ TFile.Reader.Scanner.Entry entry = aclScanner.entry();
+ key.readFields(entry.getKeyStream());
+ if (key.toString().equals(APPLICATION_ACL_KEY.toString())) {
+ DataInputStream valueStream = entry.getValueStream();
+ while (true) {
+ String appAccessOp = null;
+ String aclString = null;
+ try {
+ appAccessOp = valueStream.readUTF();
+ } catch (EOFException e) {
+ // Valid end of stream.
+ break;
+ }
+ try {
+ aclString = valueStream.readUTF();
+ } catch (EOFException e) {
+ throw new YarnRuntimeException("Error reading ACLs", e);
+ }
+ acls.put(ApplicationAccessType.valueOf(appAccessOp), aclString);
}
- acls.put(ApplicationAccessType.valueOf(appAccessOp), aclString);
}
-
+ aclScanner.advance();
}
- aclScanner.advance();
+ return acls;
+ } finally {
+ IOUtils.cleanup(LOG, aclScanner);
}
- return acls;
}
-
+
/**
* Read the next key and return the value-stream.
*
[04/48] hadoop git commit: HADOOP-12112. Make hadoop-common-project
Native code -Wall-clean (alanburlison via cmccabe)
Posted by zj...@apache.org.
HADOOP-12112. Make hadoop-common-project Native code -Wall-clean (alanburlison via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38d1678e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38d1678e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38d1678e
Branch: refs/heads/YARN-2928
Commit: 38d1678e94d5babba13693a8350140f91b02b567
Parents: 96d07d2
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 29 12:34:47 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 1 +
.../native/src/org/apache/hadoop/net/unix/DomainSocket.c | 8 ++++----
.../src/main/native/src/org/apache/hadoop/util/NativeCrc32.c | 4 ++--
4 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e8e85a0..9008ead 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12036. Consolidate all of the cmake extensions in one directory
(alanburlison via cmccabe)
+ HADOOP-12112. Make hadoop-common-project Native code -Wall-clean
+ (alanburlison via cmccabe)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index 071d830..bc78ab2 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -592,6 +592,7 @@ done:
#else
THROW(env, "java/io/IOException",
"The function Windows.createFileWithMode0() is not supported on this platform");
+ return NULL;
#endif
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
index 55ab0c3..a3f27ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
@@ -112,9 +112,9 @@ static jthrowable newSocketException(JNIEnv *env, int errnum,
* to the heap if necessary.
*/
struct flexibleBuffer {
- int8_t *curBuf;
- int8_t *allocBuf;
- int8_t stackBuf[8196];
+ jbyte *curBuf;
+ jbyte *allocBuf;
+ jbyte stackBuf[8196];
};
static jthrowable flexBufInit(JNIEnv *env, struct flexibleBuffer *flexBuf, jint length)
@@ -637,7 +637,7 @@ JNIEnv *env, jclass clazz, jint fd)
* @return NULL on success; or the unraised exception representing
* the problem.
*/
-static jthrowable write_fully(JNIEnv *env, int fd, int8_t *buf, int amt)
+static jthrowable write_fully(JNIEnv *env, int fd, jbyte *buf, int amt)
{
int err, res;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
index 74e09e6..3fceb02 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
@@ -168,7 +168,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChun
// Setup complete. Actually verify checksums.
ret = bulk_crc(data, data_len, sums, crc_type,
bytes_per_checksum, verify ? &error_data : NULL);
- if (likely(verify && ret == CHECKSUMS_VALID || !verify && ret == 0)) {
+ if (likely((verify && ret == CHECKSUMS_VALID) || (!verify && ret == 0))) {
return;
} else if (unlikely(verify && ret == INVALID_CHECKSUM_DETECTED)) {
long pos = base_pos + (error_data.bad_data - data);
@@ -261,7 +261,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChun
env, error_data.got_crc, error_data.expected_crc,
j_filename, pos);
return;
- } else if (unlikely(verify && ret != CHECKSUMS_VALID || !verify && ret != 0)) {
+ } else if (unlikely((verify && ret != CHECKSUMS_VALID) || (!verify && ret != 0))) {
THROW(env, "java/lang/AssertionError",
"Bad response code from native bulk_crc");
return;
[38/48] hadoop git commit: HADOOP-12171. Shorten overly-long htrace
span names for server (cmccabe)
Posted by zj...@apache.org.
HADOOP-12171. Shorten overly-long htrace span names for server (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f530f52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f530f52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f530f52
Branch: refs/heads/YARN-2928
Commit: 6f530f52f0a4faf7887d36e2f1a7c70b1fb4e1fc
Parents: d5192ca
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Jul 1 17:57:11 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../org/apache/hadoop/ipc/RpcClientUtil.java | 24 ++++++++++++++++++++
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +++-
.../org/apache/hadoop/tracing/TestTracing.java | 18 +++++++--------
4 files changed, 38 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 39e2e5e..24431ba 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -673,6 +673,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12124. Add HTrace support for FsShell (cmccabe)
+ HADOOP-12171. Shorten overly-long htrace span names for server (cmccabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
index d9bd71b..da1e699 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
@@ -210,4 +210,28 @@ public class RpcClientUtil {
}
return clazz.getSimpleName() + "#" + method.getName();
}
+
+ /**
+ * Convert an RPC class method to a string.
+ * The format we want is
+ * 'SecondOutermostClassShortName#OutermostClassShortName'.
+ *
+ * For example, if the full class name is:
+ * org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations
+ *
+ * the format we want is:
+ * ClientProtocol#getBlockLocations
+ */
+ public static String toTraceName(String fullName) {
+ int lastPeriod = fullName.lastIndexOf('.');
+ if (lastPeriod < 0) {
+ return fullName;
+ }
+ int secondLastPeriod = fullName.lastIndexOf('.', lastPeriod - 1);
+ if (secondLastPeriod < 0) {
+ return fullName;
+ }
+ return fullName.substring(secondLastPeriod + 1, lastPeriod) + "#" +
+ fullName.substring(lastPeriod + 1);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 98fffc0..4026fe0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -1963,7 +1963,9 @@ public abstract class Server {
// If the incoming RPC included tracing info, always continue the trace
TraceInfo parentSpan = new TraceInfo(header.getTraceInfo().getTraceId(),
header.getTraceInfo().getParentId());
- traceSpan = Trace.startSpan(rpcRequest.toString(), parentSpan).detach();
+ traceSpan = Trace.startSpan(
+ RpcClientUtil.toTraceName(rpcRequest.toString()),
+ parentSpan).detach();
}
Call call = new Call(header.getCallId(), header.getRetryCount(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 58b3659..c3d2c73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -67,18 +67,18 @@ public class TestTracing {
String[] expectedSpanNames = {
"testWriteTraceHooks",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.create",
+ "ClientProtocol#create",
"ClientNamenodeProtocol#create",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.fsync",
+ "ClientProtocol#fsync",
"ClientNamenodeProtocol#fsync",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
+ "ClientProtocol#complete",
"ClientNamenodeProtocol#complete",
"newStreamForCreate",
"DFSOutputStream#write",
"DFSOutputStream#close",
"dataStreamer",
"OpWriteBlockProto",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.addBlock",
+ "ClientProtocol#addBlock",
"ClientNamenodeProtocol#addBlock"
};
SetSpanReceiver.assertSpanNamesFound(expectedSpanNames);
@@ -95,11 +95,11 @@ public class TestTracing {
// and children of them are exception.
String[] spansInTopTrace = {
"testWriteTraceHooks",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.create",
+ "ClientProtocol#create",
"ClientNamenodeProtocol#create",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.fsync",
+ "ClientProtocol#fsync",
"ClientNamenodeProtocol#fsync",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
+ "ClientProtocol#complete",
"ClientNamenodeProtocol#complete",
"newStreamForCreate",
"DFSOutputStream#write",
@@ -113,7 +113,7 @@ public class TestTracing {
// test for timeline annotation added by HADOOP-11242
Assert.assertEquals("called",
- map.get("org.apache.hadoop.hdfs.protocol.ClientProtocol.create")
+ map.get("ClientProtocol#create")
.get(0).getTimelineAnnotations()
.get(0).getMessage());
@@ -131,7 +131,7 @@ public class TestTracing {
String[] expectedSpanNames = {
"testReadTraceHooks",
- "org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations",
+ "ClientProtocol#getBlockLocations",
"ClientNamenodeProtocol#getBlockLocations",
"OpReadBlockProto"
};
[21/48] hadoop git commit: HADOOP-12159. Move DistCpUtils#compareFs()
to org.apache.hadoop.fs.FileUtil and fix for HA namespaces (rchiang via
rkanter)
Posted by zj...@apache.org.
HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil and fix for HA namespaces (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c906166f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c906166f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c906166f
Branch: refs/heads/YARN-2928
Commit: c906166f7ebaa851283f391b62be50976c71a2bd
Parents: 349b9f9
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 30 16:42:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/FileUtil.java | 42 +++++++
.../java/org/apache/hadoop/fs/TestFileUtil.java | 112 ++++++++++++++++++-
.../java/org/apache/hadoop/tools/DistCp.java | 3 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 39 -------
5 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 19ba620..4e5f6bb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -910,6 +910,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10798. globStatus() should always return a sorted list of files
(cmccabe)
+ HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil
+ and fix for HA namespaces (rchiang via rkanter)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 9b9e213..8abb4eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
@@ -1329,4 +1332,43 @@ public class FileUtil {
unexpandedWildcardClasspath.toString()};
return jarCp;
}
+
+ public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
+ if (srcFs==null || destFs==null) {
+ return false;
+ }
+ URI srcUri = srcFs.getUri();
+ URI dstUri = destFs.getUri();
+ if (srcUri.getScheme()==null) {
+ return false;
+ }
+ if (!srcUri.getScheme().equals(dstUri.getScheme())) {
+ return false;
+ }
+ String srcHost = srcUri.getHost();
+ String dstHost = dstUri.getHost();
+ if ((srcHost!=null) && (dstHost!=null)) {
+ if (srcHost.equals(dstHost)) {
+ return srcUri.getPort()==dstUri.getPort();
+ }
+ try {
+ srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
+ dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
+ } catch (UnknownHostException ue) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not compare file-systems. Unknown host: ", ue);
+ }
+ return false;
+ }
+ if (!srcHost.equals(dstHost)) {
+ return false;
+ }
+ } else if (srcHost==null && dstHost!=null) {
+ return false;
+ } else if (srcHost!=null) {
+ return false;
+ }
+ // check for ports
+ return srcUri.getPort()==dstUri.getPort();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 41794b8..3418ade 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -17,7 +17,8 @@
*/
package org.apache.hadoop.fs;
-import org.junit.Before;
+import org.junit.*;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -25,8 +26,11 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
+import java.net.InetAddress;
import java.net.URI;
import java.io.PrintWriter;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -44,10 +48,12 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarOutputStream;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
+
+import javax.print.attribute.URISyntax;
+
import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
public class TestFileUtil {
private static final Log LOG = LogFactory.getLog(TestFileUtil.class);
@@ -64,6 +70,25 @@ public class TestFileUtil {
private final File dir2 = new File(del, DIR + "2");
private final File partitioned = new File(TEST_DIR, "partitioned");
+ private InetAddress inet1;
+ private InetAddress inet2;
+ private InetAddress inet3;
+ private InetAddress inet4;
+ private InetAddress inet5;
+ private InetAddress inet6;
+ private URI uri1;
+ private URI uri2;
+ private URI uri3;
+ private URI uri4;
+ private URI uri5;
+ private URI uri6;
+ private FileSystem fs1;
+ private FileSystem fs2;
+ private FileSystem fs3;
+ private FileSystem fs4;
+ private FileSystem fs5;
+ private FileSystem fs6;
+
/**
* Creates multiple directories for testing.
*
@@ -80,6 +105,7 @@ public class TestFileUtil {
* file: part-r-00000, contents: "foo"
* file: part-r-00001, contents: "bar"
*/
+ @Ignore
private void setupDirs() throws IOException {
Assert.assertFalse(del.exists());
Assert.assertFalse(tmp.exists());
@@ -1096,4 +1122,82 @@ public class TestFileUtil {
}
}
}
+
+ @Ignore
+ public void setupCompareFs() {
+ // Set up Strings
+ String host1 = "1.2.3.4";
+ String host2 = "2.3.4.5";
+ int port1 = 7000;
+ int port2 = 7001;
+ String uris1 = "hdfs://" + host1 + ":" + Integer.toString(port1) + "/tmp/foo";
+ String uris2 = "hdfs://" + host1 + ":" + Integer.toString(port2) + "/tmp/foo";
+ String uris3 = "hdfs://" + host2 + ":" + Integer.toString(port2) + "/tmp/foo";
+ String uris4 = "hdfs://" + host2 + ":" + Integer.toString(port2) + "/tmp/foo";
+ String uris5 = "file:///" + host1 + ":" + Integer.toString(port1) + "/tmp/foo";
+ String uris6 = "hdfs:///" + host1 + "/tmp/foo";
+ // Set up URI objects
+ try {
+ uri1 = new URI(uris1);
+ uri2 = new URI(uris2);
+ uri3 = new URI(uris3);
+ uri4 = new URI(uris4);
+ uri5 = new URI(uris5);
+ uri6 = new URI(uris6);
+ } catch (URISyntaxException use) {
+ }
+ // Set up InetAddress
+ inet1 = mock(InetAddress.class);
+ when(inet1.getCanonicalHostName()).thenReturn(host1);
+ inet2 = mock(InetAddress.class);
+ when(inet2.getCanonicalHostName()).thenReturn(host1);
+ inet3 = mock(InetAddress.class);
+ when(inet3.getCanonicalHostName()).thenReturn(host2);
+ inet4 = mock(InetAddress.class);
+ when(inet4.getCanonicalHostName()).thenReturn(host2);
+ inet5 = mock(InetAddress.class);
+ when(inet5.getCanonicalHostName()).thenReturn(host1);
+ inet6 = mock(InetAddress.class);
+ when(inet6.getCanonicalHostName()).thenReturn(host1);
+
+ // Link of InetAddress to corresponding URI
+ try {
+ when(InetAddress.getByName(uris1)).thenReturn(inet1);
+ when(InetAddress.getByName(uris2)).thenReturn(inet2);
+ when(InetAddress.getByName(uris3)).thenReturn(inet3);
+ when(InetAddress.getByName(uris4)).thenReturn(inet4);
+ when(InetAddress.getByName(uris5)).thenReturn(inet5);
+ } catch (UnknownHostException ue) {
+ }
+
+ fs1 = mock(FileSystem.class);
+ when(fs1.getUri()).thenReturn(uri1);
+ fs2 = mock(FileSystem.class);
+ when(fs2.getUri()).thenReturn(uri2);
+ fs3 = mock(FileSystem.class);
+ when(fs3.getUri()).thenReturn(uri3);
+ fs4 = mock(FileSystem.class);
+ when(fs4.getUri()).thenReturn(uri4);
+ fs5 = mock(FileSystem.class);
+ when(fs5.getUri()).thenReturn(uri5);
+ fs6 = mock(FileSystem.class);
+ when(fs6.getUri()).thenReturn(uri6);
+ }
+
+ @Test
+ public void testCompareFsNull() throws Exception {
+ setupCompareFs();
+ assertEquals(FileUtil.compareFs(null,fs1),false);
+ assertEquals(FileUtil.compareFs(fs1,null),false);
+ }
+
+ @Test
+ public void testCompareFsDirectories() throws Exception {
+ setupCompareFs();
+ assertEquals(FileUtil.compareFs(fs1,fs1),true);
+ assertEquals(FileUtil.compareFs(fs1,fs2),false);
+ assertEquals(FileUtil.compareFs(fs1,fs5),false);
+ assertEquals(FileUtil.compareFs(fs3,fs4),true);
+ assertEquals(FileUtil.compareFs(fs1,fs6),false);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 6921a1e..2385df3 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Cluster;
@@ -347,7 +348,7 @@ public class DistCp extends Configured implements Tool {
workDir = new Path(workDir, WIP_PREFIX + targetPath.getName()
+ rand.nextInt());
FileSystem workFS = workDir.getFileSystem(configuration);
- if (!DistCpUtils.compareFs(targetFS, workFS)) {
+ if (!FileUtil.compareFs(targetFS, workFS)) {
throw new IllegalArgumentException("Work path " + workDir +
" and target path " + targetPath + " are in different file system");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index 5ac5bf1..70d947e 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -468,43 +468,4 @@ public class DistCpUtils {
return (sourceChecksum == null || targetChecksum == null ||
sourceChecksum.equals(targetChecksum));
}
-
- /* see if two file systems are the same or not
- *
- */
- public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
- URI srcUri = srcFs.getUri();
- URI dstUri = destFs.getUri();
- if (srcUri.getScheme() == null) {
- return false;
- }
- if (!srcUri.getScheme().equals(dstUri.getScheme())) {
- return false;
- }
- String srcHost = srcUri.getHost();
- String dstHost = dstUri.getHost();
- if ((srcHost != null) && (dstHost != null)) {
- try {
- srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
- dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
- } catch(UnknownHostException ue) {
- if (LOG.isDebugEnabled())
- LOG.debug("Could not compare file-systems. Unknown host: ", ue);
- return false;
- }
- if (!srcHost.equals(dstHost)) {
- return false;
- }
- }
- else if (srcHost == null && dstHost != null) {
- return false;
- }
- else if (srcHost != null) {
- return false;
- }
-
- //check for ports
-
- return srcUri.getPort() == dstUri.getPort();
- }
}
[25/48] hadoop git commit: HADOOP-10798. globStatus() should always
return a sorted list of files (cmccabe)
Posted by zj...@apache.org.
HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52d4f05a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52d4f05a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52d4f05a
Branch: refs/heads/YARN-2928
Commit: 52d4f05a83024cbe78eff6c58048c9595261a27f
Parents: 0e713f2
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:40:21 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/Globber.java | 11 ++++++++-
.../org/apache/hadoop/fs/TestGlobPaths.java | 25 ++++++++++++++++++++
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2aab8b4..4d69a229 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
(J.Andreina via vinayakumarb)
+ HADOOP-10798. globStatus() should always return a sorted list of files
+ (cmccabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index 8a8137a..9cb810f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.LogFactory;
@@ -285,6 +286,14 @@ class Globber {
(flattenedPatterns.size() <= 1)) {
return null;
}
- return results.toArray(new FileStatus[0]);
+ /*
+ * In general, the results list will already be sorted, since listStatus
+ * returns results in sorted order for many Hadoop filesystems. However,
+ * not all Hadoop filesystems have this property. So we sort here in order
+ * to get consistent results. See HADOOP-10798 for details.
+ */
+ FileStatus ret[] = results.toArray(new FileStatus[0]);
+ Arrays.sort(ret);
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
index 50e2e5b..afd8fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
@@ -21,9 +21,11 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.UUID;
import java.util.regex.Pattern;
+import com.google.common.collect.Ordering;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -1284,4 +1286,27 @@ public class TestGlobPaths {
public void testNonTerminalGlobsOnFC() throws Exception {
testOnFileContext(new TestNonTerminalGlobs(true));
}
+
+ @Test
+ public void testLocalFilesystem() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ String localTmp = System.getProperty("java.io.tmpdir");
+ Path base = new Path(new Path(localTmp), UUID.randomUUID().toString());
+ Assert.assertTrue(fs.mkdirs(base));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "e")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "c")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "a")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "d")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "b")));
+ fs.deleteOnExit(base);
+ FileStatus[] status = fs.globStatus(new Path(base, "*"));
+ ArrayList list = new ArrayList();
+ for (FileStatus f: status) {
+ list.add(f.getPath().toString());
+ }
+ boolean sorted = Ordering.natural().isOrdered(list);
+ Assert.assertTrue(sorted);
+ }
}
+
[42/48] hadoop git commit: HADOOP-12172. FsShell mkdir -p makes an
unnecessary check for the existence of the parent. Contributed by Chris
Nauroth.
Posted by zj...@apache.org.
HADOOP-12172. FsShell mkdir -p makes an unnecessary check for the existence of the parent. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8a65745
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8a65745
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8a65745
Branch: refs/heads/YARN-2928
Commit: a8a65745c40bc662065e140664062ca7dad3b9f2
Parents: 6f530f5
Author: cnauroth <cn...@apache.org>
Authored: Wed Jul 1 19:47:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/shell/Mkdir.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8a65745/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 24431ba..312a996 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -704,6 +704,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12112. Make hadoop-common-project Native code -Wall-clean
(alanburlison via cmccabe)
+ HADOOP-12172. FsShell mkdir -p makes an unnecessary check for the existence
+ of the parent. (cnauroth)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8a65745/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
index 74bad62..9f39da2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
@@ -70,7 +70,8 @@ class Mkdir extends FsCommand {
protected void processNonexistentPath(PathData item) throws IOException {
// check if parent exists. this is complicated because getParent(a/b/c/) returns a/b/c, but
// we want a/b
- if (!item.fs.exists(new Path(item.path.toString()).getParent()) && !createParents) {
+ if (!createParents &&
+ !item.fs.exists(new Path(item.path.toString()).getParent())) {
throw new PathNotFoundException(item.toString());
}
if (!item.fs.mkdirs(item.path)) {
[12/48] hadoop git commit: HDFS-8627. NPE thrown if unable to fetch
token from Namenode (Contributed by J.Andreina)
Posted by zj...@apache.org.
HDFS-8627. NPE thrown if unable to fetch token from Namenode (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9634540
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9634540
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9634540
Branch: refs/heads/YARN-2928
Commit: d963454094e50af4aa7db4dbefe90652cd768a92
Parents: c9ae815
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:42:59 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/tools/DelegationTokenFetcher.java | 19 +++++++++++--------
.../hdfs/tools/TestDelegationTokenFetcher.java | 15 +++++++++++++++
3 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9634540/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ea67ea1..b065f98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -338,6 +338,9 @@ Trunk (Unreleased)
HDFS-8412. Fix the test failures in HTTPFS. (umamahesh)
+ HDFS-8627. NPE thrown if unable to fetch token from Namenode
+ (J.Andreina via vinayakumarb)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9634540/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 6376243..803402d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -176,14 +176,17 @@ public class DelegationTokenFetcher {
final String renewer, final Path tokenFile)
throws IOException {
Token<?> token = fs.getDelegationToken(renewer);
-
- Credentials cred = new Credentials();
- cred.addToken(token.getKind(), token);
- cred.writeTokenStorageFile(tokenFile, conf);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Fetched token " + fs.getUri() + " for " + token.getService()
- + " into " + tokenFile);
+ if (null != token) {
+ Credentials cred = new Credentials();
+ cred.addToken(token.getKind(), token);
+ cred.writeTokenStorageFile(tokenFile, conf);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Fetched token " + fs.getUri() + " for " +
+ token.getService() + " into " + tokenFile);
+ }
+ } else {
+ System.err.println("ERROR: Failed to fetch token from " + fs.getUri());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9634540/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
index b4cf1c0..ab3933b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
@@ -90,4 +90,19 @@ public class TestDelegationTokenFetcher {
DelegationTokenFetcher.cancelTokens(conf, p);
Assert.assertEquals(testToken, FakeRenewer.getLastCanceled());
}
+
+ /**
+ * If token returned is null, saveDelegationToken should not
+ * throw nullPointerException
+ */
+ @Test
+ public void testReturnedTokenIsNull() throws Exception {
+ WebHdfsFileSystem fs = mock(WebHdfsFileSystem.class);
+ doReturn(null).when(fs).getDelegationToken(anyString());
+ Path p = new Path(f.getRoot().getAbsolutePath(), tokenFile);
+ DelegationTokenFetcher.saveDelegationToken(conf, fs, null, p);
+ // When Token returned is null, TokenFile should not exist
+ Assert.assertFalse(p.getFileSystem(conf).exists(p));
+
+ }
}
[14/48] hadoop git commit: HDFS-8579. Update HDFS usage with missing
options (Contributed by J.Andreina)
Posted by zj...@apache.org.
HDFS-8579. Update HDFS usage with missing options (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a943699d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a943699d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a943699d
Branch: refs/heads/YARN-2928
Commit: a943699dce1797a148c8f0c72f5dffe6af92a777
Parents: 4e39b96
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:00:22 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a943699d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3535f90..ea67ea1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -983,6 +983,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Brahma
Reddy Battula via Arpit Agarwal)
+ HDFS-8579. Update HDFS usage with missing options
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a943699d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index f464261..2e09cef 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -26,6 +26,7 @@ function hadoop_usage
echo " Hadoop jar and the required libraries"
echo " crypto configure HDFS encryption zones"
echo " datanode run a DFS datanode"
+ echo " debug run a Debug Admin to execute HDFS debug commands"
echo " dfs run a filesystem command on the file system"
echo " dfsadmin run a DFS admin client"
echo " fetchdt fetch a delegation token from the NameNode"
[41/48] hadoop git commit: HADOOP-12173. NetworkTopology::add calls
toString always. Contributed by Inigo Goiri
Posted by zj...@apache.org.
HADOOP-12173. NetworkTopology::add calls toString always. Contributed by Inigo Goiri
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/513eb8b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/513eb8b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/513eb8b6
Branch: refs/heads/YARN-2928
Commit: 513eb8b67659b96510051a65faca7a55164bb291
Parents: 0823b93
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Jul 2 21:39:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/net/NetworkTopology.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/513eb8b6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index c60cc0b..63b6763 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -396,14 +396,13 @@ public class NetworkTopology {
int newDepth = NodeBase.locationToDepth(node.getNetworkLocation()) + 1;
netlock.writeLock().lock();
try {
- String oldTopoStr = this.toString();
if( node instanceof InnerNode ) {
throw new IllegalArgumentException(
"Not allow to add an inner node: "+NodeBase.getPath(node));
}
if ((depthOfAllLeaves != -1) && (depthOfAllLeaves != newDepth)) {
LOG.error("Error: can't add leaf node " + NodeBase.getPath(node) +
- " at depth " + newDepth + " to topology:\n" + oldTopoStr);
+ " at depth " + newDepth + " to topology:\n" + this.toString());
throw new InvalidTopologyException("Failed to add " + NodeBase.getPath(node) +
": You cannot have a rack and a non-rack node at the same " +
"level of the network topology.");