You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/07/06 22:57:23 UTC

[01/48] hadoop git commit: HDFS-7390. Provide JMX metrics per storage type. (Benoy Antony)

Repository: hadoop
Updated Branches:
  refs/heads/YARN-2928 18c48592e -> 683755269


HDFS-7390. Provide JMX metrics per storage type. (Benoy Antony)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/130d7d1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/130d7d1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/130d7d1b

Branch: refs/heads/YARN-2928
Commit: 130d7d1b644cf61e295dfbf1654ffb82f1eabe56
Parents: 18c4859
Author: Benoy Antony <be...@apache.org>
Authored: Mon Jun 29 11:00:22 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:00 2015 -0700

----------------------------------------------------------------------
 .../server/blockmanagement/BlockManager.java    |  15 +-
 .../blockmanagement/BlockStatsMXBean.java       |  36 +++++
 .../blockmanagement/DatanodeStatistics.java     |   6 +
 .../blockmanagement/HeartbeatManager.java       |  96 +++++++++++-
 .../blockmanagement/StorageTypeStats.java       | 115 +++++++++++++++
 .../blockmanagement/TestBlockStatsMXBean.java   | 146 +++++++++++++++++++
 6 files changed, 412 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5bd4980..0b60a97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -39,6 +39,8 @@ import java.util.TreeSet;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicLong;
 
+import javax.management.ObjectName;
+
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -85,6 +87,7 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
 import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
+import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Daemon;
@@ -94,6 +97,7 @@ import org.apache.hadoop.util.Time;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Sets;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,7 +105,7 @@ import org.slf4j.LoggerFactory;
  * Keeps information related to the blocks stored in the Hadoop cluster.
  */
 @InterfaceAudience.Private
-public class BlockManager {
+public class BlockManager implements BlockStatsMXBean {
 
   public static final Logger LOG = LoggerFactory.getLogger(BlockManager.class);
   public static final Logger blockLog = NameNode.blockStateChangeLog;
@@ -129,6 +133,7 @@ public class BlockManager {
   private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   private final BlockReportLeaseManager blockReportLeaseManager;
+  private ObjectName mxBeanName;
 
   /** Used by metrics */
   public long getPendingReplicationBlocksCount() {
@@ -468,6 +473,7 @@ public class BlockManager {
     pendingReplications.start();
     datanodeManager.activate(conf);
     this.replicationThread.start();
+    mxBeanName = MBeans.register("NameNode", "BlockStats", this);
   }
 
   public void close() {
@@ -3944,6 +3950,8 @@ public class BlockManager {
   public void shutdown() {
     stopReplicationInitializer();
     blocksMap.close();
+    MBeans.unregister(mxBeanName);
+    mxBeanName = null;
   }
   
   public void clear() {
@@ -3954,4 +3962,9 @@ public class BlockManager {
   public BlockReportLeaseManager getBlockReportLeaseManager() {
     return blockReportLeaseManager;
   }
+
+  @Override // BlockStatsMXBean
+  public Map<StorageType, StorageTypeStats> getStorageTypeStats() {
+    return  datanodeManager.getDatanodeStatistics().getStorageTypeStats();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
new file mode 100644
index 0000000..f22c537
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
+
+/**
+ * This is an interface used to retrieve statistic information related to
+ * block management.
+ */
+public interface BlockStatsMXBean {
+
+  /**
+   * The statistics of storage types.
+   *
+   * @return get storage statistics per storage type
+   */
+  Map<StorageType, StorageTypeStats> getStorageTypeStats();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
index c9bc3e5..33eca2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 
 /** Datanode statistics */
@@ -71,4 +74,7 @@ public interface DatanodeStatistics {
 
   /** @return the expired heartbeats */
   public int getExpiredHeartbeats();
+
+  /** @return Storage Tier statistics*/
+  Map<StorageType, StorageTypeStats> getStorageTypeStats();
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 9017fe1..cc9365d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -18,9 +18,15 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -189,6 +195,11 @@ class HeartbeatManager implements DatanodeStatistics {
     return stats.expiredHeartbeats;
   }
 
+  @Override
+  public  Map<StorageType, StorageTypeStats> getStorageTypeStats() {
+    return stats.statsMap.get();
+  }
+
   synchronized void register(final DatanodeDescriptor d) {
     if (!d.isAlive) {
       addDatanode(d);
@@ -393,6 +404,9 @@ class HeartbeatManager implements DatanodeStatistics {
    * For decommissioning/decommissioned nodes, only used capacity is counted.
    */
   private static class Stats {
+
+    private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
+
     private long capacityTotal = 0L;
     private long capacityUsed = 0L;
     private long capacityRemaining = 0L;
@@ -420,6 +434,14 @@ class HeartbeatManager implements DatanodeStatistics {
       }
       cacheCapacity += node.getCacheCapacity();
       cacheUsed += node.getCacheUsed();
+      Set<StorageType> storageTypes = new HashSet<>();
+      for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) {
+        statsMap.addStorage(storageInfo, node);
+        storageTypes.add(storageInfo.getStorageType());
+      }
+      for (StorageType storageType : storageTypes) {
+        statsMap.addNode(storageType, node);
+      }
     }
 
     private void subtract(final DatanodeDescriptor node) {
@@ -436,6 +458,14 @@ class HeartbeatManager implements DatanodeStatistics {
       }
       cacheCapacity -= node.getCacheCapacity();
       cacheUsed -= node.getCacheUsed();
+      Set<StorageType> storageTypes = new HashSet<>();
+      for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) {
+        statsMap.subtractStorage(storageInfo, node);
+        storageTypes.add(storageInfo.getStorageType());
+      }
+      for (StorageType storageType : storageTypes) {
+        statsMap.subtractNode(storageType, node);
+      }
     }
     
     /** Increment expired heartbeat counter. */
@@ -443,5 +473,69 @@ class HeartbeatManager implements DatanodeStatistics {
       expiredHeartbeats++;
     }
   }
-}
 
+  /** StorageType specific statistics.
+   * For decommissioning/decommissioned nodes, only used capacity is counted.
+   */
+
+  static final class StorageTypeStatsMap {
+
+    private Map<StorageType, StorageTypeStats> storageTypeStatsMap =
+          new IdentityHashMap<>();
+
+    private StorageTypeStatsMap() {}
+
+    private StorageTypeStatsMap(StorageTypeStatsMap other) {
+      storageTypeStatsMap =
+          new IdentityHashMap<>(other.storageTypeStatsMap);
+      for (Map.Entry<StorageType, StorageTypeStats> entry :
+          storageTypeStatsMap.entrySet()) {
+        entry.setValue(new StorageTypeStats(entry.getValue()));
+      }
+    }
+
+    private Map<StorageType, StorageTypeStats> get() {
+      return Collections.unmodifiableMap(storageTypeStatsMap);
+    }
+
+    private void addNode(StorageType storageType,
+        final DatanodeDescriptor node) {
+      StorageTypeStats storageTypeStats =
+          storageTypeStatsMap.get(storageType);
+      if (storageTypeStats == null) {
+        storageTypeStats = new StorageTypeStats();
+        storageTypeStatsMap.put(storageType, storageTypeStats);
+      }
+      storageTypeStats.addNode(node);
+    }
+
+    private void addStorage(final DatanodeStorageInfo info,
+        final DatanodeDescriptor node) {
+      StorageTypeStats storageTypeStats =
+          storageTypeStatsMap.get(info.getStorageType());
+      if (storageTypeStats == null) {
+        storageTypeStats = new StorageTypeStats();
+        storageTypeStatsMap.put(info.getStorageType(), storageTypeStats);
+      }
+      storageTypeStats.addStorage(info, node);
+    }
+
+    private void subtractStorage(final DatanodeStorageInfo info,
+        final DatanodeDescriptor node) {
+      StorageTypeStats storageTypeStats =
+          storageTypeStatsMap.get(info.getStorageType());
+      if (storageTypeStats != null) {
+        storageTypeStats.subtractStorage(info, node);
+      }
+    }
+
+    private void subtractNode(StorageType storageType,
+        final DatanodeDescriptor node) {
+      StorageTypeStats storageTypeStats =
+          storageTypeStatsMap.get(storageType);
+      if (storageTypeStats != null) {
+        storageTypeStats.subtractNode(node);
+      }
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
new file mode 100644
index 0000000..45dcc8d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.beans.ConstructorProperties;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Statistics per StorageType.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class StorageTypeStats {
+  private long capacityTotal = 0L;
+  private long capacityUsed = 0L;
+  private long capacityRemaining = 0L;
+  private long blockPoolUsed = 0L;
+  private int nodesInService = 0;
+
+  @ConstructorProperties({"capacityTotal",
+      "capacityUsed", "capacityRemaining",  "blockPoolUsed", "nodesInService"})
+  public StorageTypeStats(long capacityTotal, long capacityUsed,
+      long capacityRemaining, long blockPoolUsed, int nodesInService) {
+    this.capacityTotal = capacityTotal;
+    this.capacityUsed = capacityUsed;
+    this.capacityRemaining = capacityRemaining;
+    this.blockPoolUsed = blockPoolUsed;
+    this.nodesInService = nodesInService;
+  }
+
+  public long getCapacityTotal() {
+    return capacityTotal;
+  }
+
+  public long getCapacityUsed() {
+    return capacityUsed;
+  }
+
+  public long getCapacityRemaining() {
+    return capacityRemaining;
+  }
+
+  public long getBlockPoolUsed() {
+    return blockPoolUsed;
+  }
+
+  public int getNodesInService() {
+    return nodesInService;
+  }
+
+  StorageTypeStats() {}
+
+  StorageTypeStats(StorageTypeStats other) {
+    capacityTotal = other.capacityTotal;
+    capacityUsed = other.capacityUsed;
+    capacityRemaining = other.capacityRemaining;
+    blockPoolUsed = other.blockPoolUsed;
+    nodesInService = other.nodesInService;
+  }
+
+  void addStorage(final DatanodeStorageInfo info,
+      final DatanodeDescriptor node) {
+    capacityUsed += info.getDfsUsed();
+    blockPoolUsed += info.getBlockPoolUsed();
+    if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+      capacityTotal += info.getCapacity();
+      capacityRemaining += info.getRemaining();
+    } else {
+      capacityTotal += info.getDfsUsed();
+    }
+  }
+
+  void addNode(final DatanodeDescriptor node) {
+    if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+      nodesInService++;
+    }
+  }
+
+  void subtractStorage(final DatanodeStorageInfo info,
+      final DatanodeDescriptor node) {
+    capacityUsed -= info.getDfsUsed();
+    blockPoolUsed -= info.getBlockPoolUsed();
+    if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+      capacityTotal -= info.getCapacity();
+      capacityRemaining -= info.getRemaining();
+    } else {
+      capacityTotal -= info.getDfsUsed();
+    }
+  }
+
+  void subtractNode(final DatanodeDescriptor node) {
+    if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+      nodesInService--;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/130d7d1b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
new file mode 100644
index 0000000..43d983d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
+
+/**
+ * Class for testing {@link BlockStatsMXBean} implementation
+ */
+public class TestBlockStatsMXBean {
+
+  private MiniDFSCluster cluster;
+
+  @Before
+  public void setup() throws IOException {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    cluster = null;
+    StorageType[][] types = new StorageType[6][];
+    for (int i=0; i<3; i++) {
+      types[i] = new StorageType[] {StorageType.RAM_DISK, StorageType.DISK};
+    }
+    for (int i=3; i< 5; i++) {
+      types[i] = new StorageType[] {StorageType.RAM_DISK, StorageType.ARCHIVE};
+    }
+    types[5] = new StorageType[] {StorageType.RAM_DISK, StorageType.ARCHIVE,
+        StorageType.ARCHIVE};
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(6).
+        storageTypes(types).storagesPerDatanode(3).build();
+    cluster.waitActive();
+  }
+
+  @After
+  public void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Test
+  public void testStorageTypeStats() throws Exception {
+    Map<StorageType, StorageTypeStats> storageTypeStatsMap =
+        cluster.getNamesystem().getBlockManager().getStorageTypeStats();
+    assertTrue(storageTypeStatsMap.containsKey(StorageType.RAM_DISK));
+    assertTrue(storageTypeStatsMap.containsKey(StorageType.DISK));
+    assertTrue(storageTypeStatsMap.containsKey(StorageType.ARCHIVE));
+
+    StorageTypeStats storageTypeStats =
+        storageTypeStatsMap.get(StorageType.RAM_DISK);
+    assertEquals(6, storageTypeStats.getNodesInService());
+
+    storageTypeStats = storageTypeStatsMap.get(StorageType.DISK);
+    assertEquals(3, storageTypeStats.getNodesInService());
+
+    storageTypeStats = storageTypeStatsMap.get(StorageType.ARCHIVE);
+    assertEquals(3, storageTypeStats.getNodesInService());
+  }
+
+  protected static String readOutput(URL url) throws IOException {
+    StringBuilder out = new StringBuilder();
+    InputStream in = url.openConnection().getInputStream();
+    byte[] buffer = new byte[64 * 1024];
+    int len = in.read(buffer);
+    while (len > 0) {
+      out.append(new String(buffer, 0, len));
+      len = in.read(buffer);
+    }
+    return out.toString();
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testStorageTypeStatsJMX() throws Exception {
+    URL baseUrl = new URL (cluster.getHttpUri(0));
+    String result = readOutput(new URL(baseUrl, "/jmx"));
+    System.out.println(result);
+
+    Map<String, Object> stat = (Map<String, Object>) JSON.parse(result);
+    Object[] beans =(Object[]) stat.get("beans");
+    Map<String, Object> blockStats  = null;
+    for (Object bean : beans) {
+      Map<String, Object> map = (Map<String, Object>) bean;
+      if (map.get("name").equals("Hadoop:service=NameNode,name=BlockStats")) {
+        blockStats = map;
+      }
+    }
+    assertNotNull(blockStats);
+    Object[] storageTypeStatsList =
+        (Object[])blockStats.get("StorageTypeStats");
+    assertNotNull(storageTypeStatsList);
+    assertEquals (3, storageTypeStatsList.length);
+
+    Set<String> typesPresent = new HashSet<> ();
+    for (Object obj : storageTypeStatsList) {
+      Map<String, Object> entry = (Map<String, Object>)obj;
+      String storageType = (String)entry.get("key");
+      Map<String,Object> storageTypeStats = (Map<String,Object>)entry.get("value");
+      typesPresent.add(storageType);
+      if (storageType.equals("ARCHIVE") || storageType.equals("DISK") ) {
+        assertEquals(3l, storageTypeStats.get("nodesInService"));
+      } else if (storageType.equals("RAM_DISK")) {
+        assertEquals(6l, storageTypeStats.get("nodesInService"));
+      }
+      else {
+        fail();
+      }
+    }
+
+    assertTrue(typesPresent.contains("ARCHIVE"));
+    assertTrue(typesPresent.contains("DISK"));
+    assertTrue(typesPresent.contains("RAM_DISK"));
+  }
+}


[13/48] hadoop git commit: HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina)

Posted by zj...@apache.org.
HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9ae815f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9ae815f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9ae815f

Branch: refs/heads/YARN-2928
Commit: c9ae815f6a67e8f3150b4dd4f209a63d7da97b89
Parents: a943699
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:25:20 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../java/org/apache/hadoop/fs/FileSystem.java   |  6 ++---
 .../org/apache/hadoop/hdfs/TestDFSShell.java    | 26 ++++++++++++++++++++
 3 files changed, 32 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 50192ae..2cf9509 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -899,6 +899,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12089. StorageException complaining " no lease ID" when updating
     FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
 
+    HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
+    (J.Andreina via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 1d7bc87..c98074a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2085,9 +2085,9 @@ public abstract class FileSystem extends Configured implements Closeable {
   /** Return the total size of all files in the filesystem.*/
   public long getUsed() throws IOException{
     long used = 0;
-    FileStatus[] files = listStatus(new Path("/"));
-    for(FileStatus file:files){
-      used += file.getLen();
+    RemoteIterator<LocatedFileStatus> files = listFiles(new Path("/"), true);
+    while (files.hasNext()) {
+      used += files.next().getLen();
     }
     return used;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 2df31c4..1386124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -913,6 +913,32 @@ public class TestDFSShell {
       cluster.shutdown();
     }
   }
+
+  @Test(timeout = 30000)
+  public void testTotalSizeOfAllFiles() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+      FileSystem fs = cluster.getFileSystem();
+      // create file under root
+      FSDataOutputStream File1 = fs.create(new Path("/File1"));
+      File1.write("hi".getBytes());
+      File1.close();
+      // create file under sub-folder
+      FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2"));
+      File2.write("hi".getBytes());
+      File2.close();
+      // getUsed() should return total length of all the files in Filesystem
+      assertEquals(4, fs.getUsed());
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+        cluster = null;
+      }
+    }
+  }
+
   private static void runCount(String path, long dirs, long files, FsShell shell
     ) throws IOException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();


[19/48] hadoop git commit: MAPREDUCE-6384. Add the last reporting reducer info for too many fetch failure diagnostics. Contributed by Chang Li

Posted by zj...@apache.org.
MAPREDUCE-6384. Add the last reporting reducer info for too many fetch failure diagnostics. Contributed by Chang Li


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9465dc4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9465dc4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9465dc4c

Branch: refs/heads/YARN-2928
Commit: 9465dc4ce7f5e07fb868f61f43357b676b2f5e84
Parents: c68da70
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Jun 30 21:22:30 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |   3 +
 .../event/JobTaskAttemptFetchFailureEvent.java  |   9 +-
 .../TaskAttemptTooManyFetchFailureEvent.java    |  50 +++++++
 .../mapreduce/v2/app/job/impl/JobImpl.java      |   7 +-
 .../v2/app/job/impl/TaskAttemptImpl.java        |  15 +-
 .../mapreduce/v2/app/TestFetchFailure.java      |  31 ++--
 .../v2/app/job/impl/TestTaskAttempt.java        | 143 ++++++++++---------
 7 files changed, 168 insertions(+), 90 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index eec131c..974af61 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -388,6 +388,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6408. Queue name and user name should be printed on the job page.
     (Siqi Li via gera)
 
+    MAPREDUCE-6384. Add the last reporting reducer info for too many fetch
+    failure diagnostics (Chang Li via jlowe)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
index 37e2034..787711c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
@@ -28,13 +28,15 @@ public class JobTaskAttemptFetchFailureEvent extends JobEvent {
 
   private final TaskAttemptId reduce;
   private final List<TaskAttemptId> maps;
+  private final String hostname;
 
   public JobTaskAttemptFetchFailureEvent(TaskAttemptId reduce, 
-      List<TaskAttemptId> maps) {
-    super(reduce.getTaskId().getJobId(), 
+      List<TaskAttemptId> maps, String host) {
+    super(reduce.getTaskId().getJobId(),
         JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE);
     this.reduce = reduce;
     this.maps = maps;
+    this.hostname = host;
   }
 
   public List<TaskAttemptId> getMaps() {
@@ -45,4 +47,7 @@ public class JobTaskAttemptFetchFailureEvent extends JobEvent {
     return reduce;
   }
 
+  public String getHost() {
+    return hostname;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
new file mode 100644
index 0000000..662e712
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
@@ -0,0 +1,50 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+/**
+ * TaskAttemptTooManyFetchFailureEvent is used for TA_TOO_MANY_FETCH_FAILURE.
+ */
+public class TaskAttemptTooManyFetchFailureEvent extends TaskAttemptEvent {
+  private TaskAttemptId reduceID;
+  private String  reduceHostname;
+
+  /**
+   * Create a new TaskAttemptTooManyFetchFailureEvent.
+   * @param attemptId the id of the mapper task attempt
+   * @param reduceId the id of the reporting reduce task attempt.
+   * @param reduceHost the hostname of the reporting reduce task attempt.
+   */
+  public TaskAttemptTooManyFetchFailureEvent(TaskAttemptId attemptId,
+      TaskAttemptId reduceId, String reduceHost) {
+      super(attemptId, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE);
+    this.reduceID = reduceId;
+    this.reduceHostname = reduceHost;
+  }
+
+  public TaskAttemptId getReduceId() {
+    return reduceID;
+  }
+
+  public String getReduceHost() {
+    return reduceHostname;
+  }  
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 6e9f13c..2c48019 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -103,9 +103,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent
 import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskRecoverEvent;
@@ -1914,8 +1913,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
             && failureRate >= job.getMaxAllowedFetchFailuresFraction()) {
           LOG.info("Too many fetch-failures for output of task attempt: " + 
               mapId + " ... raising fetch failure to map");
-          job.eventHandler.handle(new TaskAttemptEvent(mapId, 
-              TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+          job.eventHandler.handle(new TaskAttemptTooManyFetchFailureEvent(mapId,
+              fetchfailureEvent.getReduce(), fetchfailureEvent.getHost()));
           job.fetchFailuresMapping.remove(mapId);
         }
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3055a25..3fa42fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptRecoverEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
 import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
@@ -1916,12 +1917,17 @@ public abstract class TaskAttemptImpl implements
     @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+      TaskAttemptTooManyFetchFailureEvent fetchFailureEvent =
+          (TaskAttemptTooManyFetchFailureEvent) event;
       // too many fetch failure can only happen for map tasks
       Preconditions
           .checkArgument(taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP);
       //add to diagnostic
-      taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
-      
+      taskAttempt.addDiagnosticInfo("Too many fetch failures."
+          + " Failing the attempt. Last failure reported by " +
+          fetchFailureEvent.getReduceId() +
+          " from host " + fetchFailureEvent.getReduceHost());
+
       if (taskAttempt.getLaunchTime() != 0) {
         taskAttempt.eventHandler
             .handle(createJobCounterUpdateEventTAFailed(taskAttempt, true));
@@ -2225,8 +2231,11 @@ public abstract class TaskAttemptImpl implements
       //this only will happen in reduce attempt type
       if (taskAttempt.reportedStatus.fetchFailedMaps != null && 
           taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) {
+        String hostname = taskAttempt.container == null ? "UNKNOWN"
+            : taskAttempt.container.getNodeId().getHost();
         taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent(
-            taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps));
+            taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps,
+                hostname));
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
index 4e4e2e7..8d25079 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
@@ -94,10 +94,10 @@ public class TestFetchFailure {
     app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);
     
     //send 3 fetch failures from reduce to trigger map re execution
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
-    
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+
     //wait for map Task state move back to RUNNING
     app.waitForState(mapTask, TaskState.RUNNING);
     
@@ -215,9 +215,9 @@ public class TestFetchFailure {
     app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);
 
     //send 3 fetch failures from reduce to trigger map re execution
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
 
     //wait for map Task state move back to RUNNING
     app.waitForState(mapTask, TaskState.RUNNING);
@@ -324,8 +324,8 @@ public class TestFetchFailure {
     updateStatus(app, reduceAttempt3, Phase.SHUFFLE);
 
     //send 2 fetch failures from reduce to prepare for map re execution
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
+    sendFetchFailure(app, reduceAttempt, mapAttempt1, "host1");
+    sendFetchFailure(app, reduceAttempt2, mapAttempt1, "host2");
 
     //We should not re-launch the map task yet
     assertEquals(TaskState.SUCCEEDED, mapTask.getState());
@@ -333,7 +333,7 @@ public class TestFetchFailure {
     updateStatus(app, reduceAttempt3, Phase.REDUCE);
 
     //send 3rd fetch failures from reduce to trigger map re execution
-    sendFetchFailure(app, reduceAttempt, mapAttempt1);
+    sendFetchFailure(app, reduceAttempt3, mapAttempt1, "host3");
 
     //wait for map Task state move back to RUNNING
     app.waitForState(mapTask, TaskState.RUNNING);
@@ -342,6 +342,11 @@ public class TestFetchFailure {
     Assert.assertEquals("Map TaskAttempt state not correct",
         TaskAttemptState.FAILED, mapAttempt1.getState());
 
+    Assert.assertEquals(mapAttempt1.getDiagnostics().get(0),
+            "Too many fetch failures. Failing the attempt. "
+            + "Last failure reported by "
+            + reduceAttempt3.getID().toString() + " from host host3");
+
     Assert.assertEquals("Num attempts in Map Task not correct",
         2, mapTask.getAttempts().size());
     
@@ -410,7 +415,6 @@ public class TestFetchFailure {
     Assert.assertEquals("Unexpected map event", convertedEvents[2],
         mapEvents[0]);
   }
-  
 
   private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) {
     TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus();
@@ -430,11 +434,12 @@ public class TestFetchFailure {
   }
 
   private void sendFetchFailure(MRApp app, TaskAttempt reduceAttempt, 
-      TaskAttempt mapAttempt) {
+      TaskAttempt mapAttempt, String hostname) {
     app.getContext().getEventHandler().handle(
         new JobTaskAttemptFetchFailureEvent(
             reduceAttempt.getID(), 
-            Arrays.asList(new TaskAttemptId[] {mapAttempt.getID()})));
+            Arrays.asList(new TaskAttemptId[] {mapAttempt.getID()}),
+                hostname));
   }
   
   static class MRAppWithHistory extends MRApp {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9465dc4c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 79b88d8..a88a935 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
 import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
@@ -507,6 +508,9 @@ public class TestTaskAttempt{
     JobId jobId = MRBuilderUtils.newJobId(appId, 1);
     TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
     TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+    TaskId reduceTaskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+    TaskAttemptId reduceTAId =
+        MRBuilderUtils.newTaskAttemptId(reduceTaskId, 0);
     Path jobFile = mock(Path.class);
 
     MockEventHandler eventHandler = new MockEventHandler();
@@ -554,8 +558,8 @@ public class TestTaskAttempt{
 
     assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
         TaskAttemptState.SUCCEEDED);
-    taImpl.handle(new TaskAttemptEvent(attemptId,
-        TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+    taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId,
+        reduceTAId, "Host"));
     assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
         TaskAttemptState.FAILED);
     taImpl.handle(new TaskAttemptEvent(attemptId,
@@ -735,72 +739,75 @@ public class TestTaskAttempt{
     
   @Test
   public void testFetchFailureAttemptFinishTime() throws Exception{
-	ApplicationId appId = ApplicationId.newInstance(1, 2);
-	ApplicationAttemptId appAttemptId =
-	ApplicationAttemptId.newInstance(appId, 0);
-	JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-	TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-	TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
-	Path jobFile = mock(Path.class);
-
-	MockEventHandler eventHandler = new MockEventHandler();
-	TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-	when(taListener.getAddress()).thenReturn(
-		new InetSocketAddress("localhost", 0));
-
-	JobConf jobConf = new JobConf();
-	jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
-	jobConf.setBoolean("fs.file.impl.disable.cache", true);
-	jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
-	jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-
-	TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
-	when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-
-	AppContext appCtx = mock(AppContext.class);
-	ClusterInfo clusterInfo = mock(ClusterInfo.class);
-	when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
-  setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
-
-	TaskAttemptImpl taImpl =
-	  new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-	  splits, jobConf, taListener,mock(Token.class), new Credentials(),
-	  new SystemClock(), appCtx);
-
-	NodeId nid = NodeId.newInstance("127.0.0.1", 0);
-	ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
-	Container container = mock(Container.class);
-	when(container.getId()).thenReturn(contId);
-	when(container.getNodeId()).thenReturn(nid);
-	when(container.getNodeHttpAddress()).thenReturn("localhost:0"); 
-	    
-	taImpl.handle(new TaskAttemptEvent(attemptId,
-	 	TaskAttemptEventType.TA_SCHEDULE));
-	taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
-	    container, mock(Map.class)));
-	taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
-	taImpl.handle(new TaskAttemptEvent(attemptId,
-	    TaskAttemptEventType.TA_DONE));
-	taImpl.handle(new TaskAttemptEvent(attemptId,
-	    TaskAttemptEventType.TA_CONTAINER_COMPLETED));
-	    
-	assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
-		      TaskAttemptState.SUCCEEDED);
-	
-	assertTrue("Task Attempt finish time is not greater than 0", 
-			taImpl.getFinishTime() > 0);
-	
-	Long finishTime = taImpl.getFinishTime();
-	Thread.sleep(5);   
-	taImpl.handle(new TaskAttemptEvent(attemptId,
-	   TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
-	
-	assertEquals("Task attempt is not in Too Many Fetch Failure state", 
-			taImpl.getState(), TaskAttemptState.FAILED);
-	
-	assertEquals("After TA_TOO_MANY_FETCH_FAILURE,"
-		+ " Task attempt finish time is not the same ",
-		finishTime, Long.valueOf(taImpl.getFinishTime()));  
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId =
+    ApplicationAttemptId.newInstance(appId, 0);
+    JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+    TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+    TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+    TaskId reducetaskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+    TaskAttemptId reduceTAId =
+        MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+    Path jobFile = mock(Path.class);
+
+    MockEventHandler eventHandler = new MockEventHandler();
+    TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+    when(taListener.getAddress()).thenReturn(
+        new InetSocketAddress("localhost", 0));
+
+    JobConf jobConf = new JobConf();
+    jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+    jobConf.setBoolean("fs.file.impl.disable.cache", true);
+    jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+    jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+    TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+    when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+    AppContext appCtx = mock(AppContext.class);
+    ClusterInfo clusterInfo = mock(ClusterInfo.class);
+    when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
+    setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
+
+    TaskAttemptImpl taImpl =
+      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+      splits, jobConf, taListener,mock(Token.class), new Credentials(),
+      new SystemClock(), appCtx);
+
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
+    Container container = mock(Container.class);
+    when(container.getId()).thenReturn(contId);
+    when(container.getNodeId()).thenReturn(nid);
+    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+    taImpl.handle(new TaskAttemptEvent(attemptId,
+        TaskAttemptEventType.TA_SCHEDULE));
+    taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+        container, mock(Map.class)));
+    taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
+    taImpl.handle(new TaskAttemptEvent(attemptId,
+        TaskAttemptEventType.TA_DONE));
+    taImpl.handle(new TaskAttemptEvent(attemptId,
+        TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+
+    assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
+        TaskAttemptState.SUCCEEDED);
+
+    assertTrue("Task Attempt finish time is not greater than 0",
+        taImpl.getFinishTime() > 0);
+
+    Long finishTime = taImpl.getFinishTime();
+    Thread.sleep(5);
+    taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId,
+        reduceTAId, "Host"));
+
+    assertEquals("Task attempt is not in Too Many Fetch Failure state",
+        taImpl.getState(), TaskAttemptState.FAILED);
+
+    assertEquals("After TA_TOO_MANY_FETCH_FAILURE,"
+        + " Task attempt finish time is not the same ",
+        finishTime, Long.valueOf(taImpl.getFinishTime()));
   }
   
   @Test


[37/48] hadoop git commit: HDFS-8666. Speedup the TestMover test. Contributed by Walter Su.

Posted by zj...@apache.org.
HDFS-8666. Speedup the TestMover test. Contributed by Walter Su.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2820f1f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2820f1f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2820f1f4

Branch: refs/heads/YARN-2928
Commit: 2820f1f4b7c4a77b9d6481e3c13611979581fed7
Parents: e556ee0
Author: Jing Zhao <ji...@apache.org>
Authored: Wed Jul 1 14:59:50 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt      |  2 ++
 .../hadoop/hdfs/server/mover/TestMover.java      | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2820f1f4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3736932..ec37542 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -694,6 +694,8 @@ Release 2.8.0 - UNRELEASED
     HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
     via Colin P. McCabe)
 
+    HDFS-8666. Speedup the TestMover tests. (Walter Su via jing9)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2820f1f4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 49e2b23..899b5c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DBlock;
 import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
 import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
+import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
 import org.apache.hadoop.hdfs.server.mover.Mover.MLocation;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -46,6 +47,21 @@ import org.junit.Assert;
 import org.junit.Test;
 
 public class TestMover {
+
+  static final int DEFAULT_BLOCK_SIZE = 100;
+
+  static {
+    TestBalancer.initTestSetup();
+  }
+
+  static void initConf(Configuration conf) {
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
+    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
+  }
+
   static Mover newMover(Configuration conf) throws IOException {
     final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
     Assert.assertEquals(1, namenodes.size());
@@ -97,6 +113,7 @@ public class TestMover {
   @Test
   public void testScheduleBlockWithinSameNode() throws Exception {
     final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(3)
         .storageTypes(
@@ -285,6 +302,7 @@ public class TestMover {
   public void testTwoReplicaSameStorageTypeShouldNotSelect() throws Exception {
     // HDFS-8147
     final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(3)
         .storageTypes(
@@ -361,6 +379,7 @@ public class TestMover {
   public void testMoverFailedRetry() throws Exception {
     // HDFS-8147
     final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
     conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2");
     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(3)


[46/48] hadoop git commit: HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync multiple times (Contributed by zhihai xu)

Posted by zj...@apache.org.
HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync multiple times (Contributed by zhihai xu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7182a03f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7182a03f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7182a03f

Branch: refs/heads/YARN-2928
Commit: 7182a03fdc23f83912b41d71f09f906f9594b0ba
Parents: b66c6cb
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jul 6 15:39:43 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 .../apache/hadoop/ha/ActiveStandbyElector.java  | 20 +++++++++++--
 .../hadoop/ha/TestActiveStandbyElector.java     | 31 ++++++++++++++++++++
 3 files changed, 51 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7182a03f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 312a996..1d737e5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -924,6 +924,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
     name using reflection. (Lei (Eddy) Xu)
 
+    HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync
+    multiple times (zhihai xu via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7182a03f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index e520a16..e458181 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -173,7 +173,9 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
 
   private Lock sessionReestablishLockForTests = new ReentrantLock();
   private boolean wantToBeInElection;
-  
+  private boolean monitorLockNodePending = false;
+  private ZooKeeper monitorLockNodeClient;
+
   /**
    * Create a new ActiveStandbyElector object <br/>
    * The elector is created by providing to it the Zookeeper configuration, the
@@ -468,7 +470,8 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
   public synchronized void processResult(int rc, String path, Object ctx,
       Stat stat) {
     if (isStaleClient(ctx)) return;
-    
+    monitorLockNodePending = false;
+
     assert wantToBeInElection :
         "Got a StatNode result after quitting election";
 
@@ -744,6 +747,11 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
     return state;
   }
 
+  @VisibleForTesting
+  synchronized boolean isMonitorLockNodePending() {
+    return monitorLockNodePending;
+  }
+
   private boolean reEstablishSession() {
     int connectionRetryCount = 0;
     boolean success = false;
@@ -949,7 +957,13 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
   }
 
   private void monitorLockNodeAsync() {
-    zkClient.exists(zkLockFilePath, 
+    if (monitorLockNodePending && monitorLockNodeClient == zkClient) {
+      LOG.info("Ignore duplicate monitor lock-node request.");
+      return;
+    }
+    monitorLockNodePending = true;
+    monitorLockNodeClient = zkClient;
+    zkClient.exists(zkLockFilePath,
         watcher, this,
         zkClient);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7182a03f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
index 2e578e2..83a3a4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
@@ -452,6 +452,10 @@ public class TestActiveStandbyElector {
         Event.KeeperState.SyncConnected);
     elector.processWatchEvent(mockZK, mockEvent);
     verifyExistCall(1);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+    elector.processResult(Code.SESSIONEXPIRED.intValue(), ZK_LOCK_NAME,
+        mockZK, new Stat());
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     // session expired should enter safe mode and initiate re-election
     // re-election checked via checking re-creation of new zookeeper and
@@ -495,6 +499,13 @@ public class TestActiveStandbyElector {
         ZK_LOCK_NAME);
     Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
     verifyExistCall(1);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+
+    Stat stat = new Stat();
+    stat.setEphemeralOwner(0L);
+    Mockito.when(mockZK.getSessionId()).thenReturn(1L);
+    elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
     Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);
@@ -504,12 +515,18 @@ public class TestActiveStandbyElector {
         Event.EventType.NodeDataChanged);
     elector.processWatchEvent(mockZK, mockEvent);
     verifyExistCall(2);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+    elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     // monitoring should be setup again after event is received
     Mockito.when(mockEvent.getType()).thenReturn(
         Event.EventType.NodeChildrenChanged);
     elector.processWatchEvent(mockZK, mockEvent);
     verifyExistCall(3);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+    elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     // lock node deletion when in standby mode should create znode again
     // successful znode creation enters active state and sets monitor
@@ -524,6 +541,10 @@ public class TestActiveStandbyElector {
         ZK_LOCK_NAME);
     Mockito.verify(mockApp, Mockito.times(1)).becomeActive();
     verifyExistCall(4);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+    stat.setEphemeralOwner(1L);
+    elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     // lock node deletion in active mode should enter neutral mode and create
     // znode again successful znode creation enters active state and sets
@@ -538,6 +559,9 @@ public class TestActiveStandbyElector {
         ZK_LOCK_NAME);
     Mockito.verify(mockApp, Mockito.times(2)).becomeActive();
     verifyExistCall(5);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+    elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     // bad path name results in fatal error
     Mockito.when(mockEvent.getPath()).thenReturn(null);
@@ -570,6 +594,13 @@ public class TestActiveStandbyElector {
         ZK_LOCK_NAME);
     Mockito.verify(mockApp, Mockito.times(1)).becomeStandby();
     verifyExistCall(1);
+    Assert.assertTrue(elector.isMonitorLockNodePending());
+
+    Stat stat = new Stat();
+    stat.setEphemeralOwner(0L);
+    Mockito.when(mockZK.getSessionId()).thenReturn(1L);
+    elector.processResult(Code.OK.intValue(), ZK_LOCK_NAME, mockZK, stat);
+    Assert.assertFalse(elector.isMonitorLockNodePending());
 
     WatchedEvent mockEvent = Mockito.mock(WatchedEvent.class);
     Mockito.when(mockEvent.getPath()).thenReturn(ZK_LOCK_NAME);


[45/48] hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong "base" parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.

Posted by zj...@apache.org.
MAPREDUCE-6425. ShuffleHandler passes wrong "base" parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b66c6cb8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b66c6cb8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b66c6cb8

Branch: refs/heads/YARN-2928
Commit: b66c6cb8be517deb7c87bf58d23dc620a63f9bd1
Parents: ccfc787
Author: Devaraj K <de...@apache.org>
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |   3 +
 .../apache/hadoop/mapred/ShuffleHandler.java    |   3 +-
 .../hadoop/mapred/TestShuffleHandler.java       | 101 +++++++++++++++++++
 3 files changed, 106 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b66c6cb8/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 68ce49b..90208ec 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -566,6 +566,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+    MAPREDUCE-6425. ShuffleHandler passes wrong "base" parameter to getMapOutputInfo
+    if mapId is not in the cache. (zhihai xu via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b66c6cb8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
         try {
           MapOutputInfo info = mapOutputInfoMap.get(mapId);
           if (info == null) {
-            info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+            info = getMapOutputInfo(outputBasePathStr + mapId,
+                mapId, reduceId, user);
           }
           lastMap =
               sendMapOutput(ctx, ch, user, mapId,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b66c6cb8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 7053653..746071f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
       Assert.assertTrue((new String(byteArr)).contains(message));
     } finally {
       shuffleHandler.stop();
+      FileUtil.fullyDelete(absLogDir);
     }
   }
 
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
     conn.disconnect();
     return rc;
   }
+
+  @Test(timeout = 100000)
+  public void testGetMapOutputInfo() throws Exception {
+    final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
+    Configuration conf = new Configuration();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+    File absLogDir = new File("target", TestShuffleHandler.class.
+        getSimpleName() + "LocDir").getAbsoluteFile();
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+    ApplicationId appId = ApplicationId.newInstance(12345, 1);
+    String appAttemptId = "attempt_12345_1_m_1_0";
+    String user = "randomUser";
+    String reducerId = "0";
+    List<File> fileMap = new ArrayList<File>();
+    createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId,
+        conf, fileMap);
+    ShuffleHandler shuffleHandler = new ShuffleHandler() {
+      @Override
+      protected Shuffle getShuffle(Configuration conf) {
+        // replace the shuffle handler with one stubbed for testing
+        return new Shuffle(conf) {
+          @Override
+          protected void populateHeaders(List<String> mapIds,
+              String outputBaseStr, String user, int reduce,
+              HttpRequest request, HttpResponse response,
+              boolean keepAliveParam, Map<String, MapOutputInfo> infoMap)
+              throws IOException {
+            // Only set response headers and skip everything else
+            // send some dummy value for content-length
+            super.setResponseHeaders(response, keepAliveParam, 100);
+          }
+          @Override
+          protected void verifyRequest(String appid,
+              ChannelHandlerContext ctx, HttpRequest request,
+              HttpResponse response, URL requestUri) throws IOException {
+            // Do nothing.
+          }
+          @Override
+          protected void sendError(ChannelHandlerContext ctx, String message,
+              HttpResponseStatus status) {
+            if (failures.size() == 0) {
+              failures.add(new Error(message));
+              ctx.getChannel().close();
+            }
+          }
+          @Override
+          protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
+              Channel ch, String user, String mapId, int reduce,
+              MapOutputInfo info) throws IOException {
+            // send a shuffle header
+            ShuffleHeader header =
+                new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
+            DataOutputBuffer dob = new DataOutputBuffer();
+            header.write(dob);
+            return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+          }
+        };
+      }
+    };
+    shuffleHandler.init(conf);
+    try {
+      shuffleHandler.start();
+      DataOutputBuffer outputBuffer = new DataOutputBuffer();
+      outputBuffer.reset();
+      Token<JobTokenIdentifier> jt =
+          new Token<JobTokenIdentifier>("identifier".getBytes(),
+          "password".getBytes(), new Text(user), new Text("shuffleService"));
+      jt.write(outputBuffer);
+      shuffleHandler
+          .initializeApplication(new ApplicationInitializationContext(user,
+          appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
+          outputBuffer.getLength())));
+      URL url =
+          new URL(
+              "http://127.0.0.1:"
+                  + shuffleHandler.getConfig().get(
+                      ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+                  + "/mapOutput?job=job_12345_0001&reduce=" + reducerId
+                  + "&map=attempt_12345_1_m_1_0");
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
+          ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+      conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
+          ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+      conn.connect();
+      try {
+        DataInputStream is = new DataInputStream(conn.getInputStream());
+        ShuffleHeader header = new ShuffleHeader();
+        header.readFields(is);
+        is.close();
+      } catch (EOFException e) {
+        // ignore
+      }
+      Assert.assertEquals(failures.size(), 0);
+    } finally {
+      shuffleHandler.stop();
+      FileUtil.fullyDelete(absLogDir);
+    }
+  }
 }


[43/48] hadoop git commit: HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch (Contributed by Vinayakumar B)

Posted by zj...@apache.org.
HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch (Contributed by Vinayakumar B)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/825dbb82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/825dbb82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/825dbb82

Branch: refs/heads/YARN-2928
Commit: 825dbb8201a2b3a35779dd802490541d426581f0
Parents: ec285b3
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jul 2 16:11:50 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |   2 +-
 .../org/apache/hadoop/hdfs/DFSInputStream.java  | 233 +++++++++++++------
 .../hadoop/hdfs/TestDFSClientRetries.java       |   2 +-
 4 files changed, 169 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ec37542..7b96c56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -696,6 +696,9 @@ Release 2.8.0 - UNRELEASED
 
     HDFS-8666. Speedup the TestMover tests. (Walter Su via jing9)
 
+    HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch
+    (vinayakumarb)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f4ceab3..4923a50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1181,7 +1181,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
     //    Get block info from namenode
     TraceScope scope = getPathTraceScope("newDFSInputStream", src);
     try {
-      return new DFSInputStream(this, src, verifyChecksum);
+      return new DFSInputStream(this, src, verifyChecksum, null);
     } finally {
       scope.close();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 6563d7b..7f3722f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -44,6 +44,7 @@ import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.ByteBufferReadable;
@@ -94,35 +95,35 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   @VisibleForTesting
   public static boolean tcpReadsDisabledForTesting = false;
   private long hedgedReadOpsLoopNumForTesting = 0;
-  private final DFSClient dfsClient;
-  private AtomicBoolean closed = new AtomicBoolean(false);
-  private final String src;
-  private final boolean verifyChecksum;
+  protected final DFSClient dfsClient;
+  protected AtomicBoolean closed = new AtomicBoolean(false);
+  protected final String src;
+  protected final boolean verifyChecksum;
 
   // state by stateful read only:
   // (protected by lock on this)
   /////
   private DatanodeInfo currentNode = null;
-  private LocatedBlock currentLocatedBlock = null;
-  private long pos = 0;
-  private long blockEnd = -1;
+  protected LocatedBlock currentLocatedBlock = null;
+  protected long pos = 0;
+  protected long blockEnd = -1;
   private BlockReader blockReader = null;
   ////
 
   // state shared by stateful and positional read:
   // (protected by lock on infoLock)
   ////
-  private LocatedBlocks locatedBlocks = null;
+  protected LocatedBlocks locatedBlocks = null;
   private long lastBlockBeingWrittenLength = 0;
   private FileEncryptionInfo fileEncryptionInfo = null;
-  private CachingStrategy cachingStrategy;
+  protected CachingStrategy cachingStrategy;
   ////
 
-  private final ReadStatistics readStatistics = new ReadStatistics();
+  protected final ReadStatistics readStatistics = new ReadStatistics();
   // lock for state shared between read and pread
   // Note: Never acquire a lock on <this> with this lock held to avoid deadlocks
   //       (it's OK to acquire this lock when the lock on <this> is held)
-  private final Object infoLock = new Object();
+  protected final Object infoLock = new Object();
 
   /**
    * Track the ByteBuffers that we have handed out to readers.
@@ -239,7 +240,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
    * back to the namenode to get a new list of block locations, and is
    * capped at maxBlockAcquireFailures
    */
-  private int failures = 0;
+  protected int failures = 0;
 
   /* XXX Use of CocurrentHashMap is temp fix. Need to fix 
    * parallel accesses to DFSInputStream (through ptreads) properly */
@@ -252,24 +253,28 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     deadNodes.put(dnInfo, dnInfo);
   }
   
-  DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum
-                 ) throws IOException, UnresolvedLinkException {
+  DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum,
+      LocatedBlocks locatedBlocks) throws IOException, UnresolvedLinkException {
     this.dfsClient = dfsClient;
     this.verifyChecksum = verifyChecksum;
     this.src = src;
     synchronized (infoLock) {
       this.cachingStrategy = dfsClient.getDefaultReadCachingStrategy();
     }
-    openInfo();
+    this.locatedBlocks = locatedBlocks;
+    openInfo(false);
   }
 
   /**
    * Grab the open-file info from namenode
+   * @param refreshLocatedBlocks whether to re-fetch locatedblocks
    */
-  void openInfo() throws IOException, UnresolvedLinkException {
+  void openInfo(boolean refreshLocatedBlocks) throws IOException,
+      UnresolvedLinkException {
     final DfsClientConf conf = dfsClient.getConf();
     synchronized(infoLock) {
-      lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
+      lastBlockBeingWrittenLength =
+          fetchLocatedBlocksAndGetLastBlockLength(refreshLocatedBlocks);
       int retriesForLastBlockLength = conf.getRetryTimesForGetLastBlockLength();
       while (retriesForLastBlockLength > 0) {
         // Getting last block length as -1 is a special case. When cluster
@@ -281,7 +286,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
               + "Datanodes might not have reported blocks completely."
               + " Will retry for " + retriesForLastBlockLength + " times");
           waitFor(conf.getRetryIntervalForGetLastBlockLength());
-          lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
+          lastBlockBeingWrittenLength =
+              fetchLocatedBlocksAndGetLastBlockLength(true);
         } else {
           break;
         }
@@ -302,8 +308,12 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     }
   }
 
-  private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException {
-    final LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0);
+  private long fetchLocatedBlocksAndGetLastBlockLength(boolean refresh)
+      throws IOException {
+    LocatedBlocks newInfo = locatedBlocks;
+    if (locatedBlocks == null || refresh) {
+      newInfo = dfsClient.getLocatedBlocks(src, 0);
+    }
     if (DFSClient.LOG.isDebugEnabled()) {
       DFSClient.LOG.debug("newInfo = " + newInfo);
     }
@@ -441,7 +451,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
    * @return located block
    * @throws IOException
    */
-  private LocatedBlock getBlockAt(long offset) throws IOException {
+  protected LocatedBlock getBlockAt(long offset) throws IOException {
     synchronized(infoLock) {
       assert (locatedBlocks != null) : "locatedBlocks is null";
 
@@ -476,7 +486,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   }
 
   /** Fetch a block from namenode and cache it */
-  private void fetchBlockAt(long offset) throws IOException {
+  protected void fetchBlockAt(long offset) throws IOException {
     synchronized(infoLock) {
       int targetBlockIdx = locatedBlocks.findBlock(offset);
       if (targetBlockIdx < 0) { // block is not cached
@@ -579,7 +589,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     }
 
     // Will be getting a new BlockReader.
-    closeCurrentBlockReader();
+    closeCurrentBlockReaders();
 
     //
     // Connect to best DataNode for desired Block, with potential offset
@@ -620,7 +630,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
         return chosenNode;
       } catch (IOException ex) {
         if (ex instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
-          DFSClient.LOG.info("Will fetch a new encryption key and retry, " 
+          DFSClient.LOG.info("Will fetch a new encryption key and retry, "
               + "encryption key was invalid when connecting to " + targetAddr
               + " : " + ex);
           // The encryption key used is invalid.
@@ -631,8 +641,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
           fetchBlockAt(target);
         } else {
           connectFailedOnce = true;
-          DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block "
-            +targetBlock.getBlock()+ ", add to deadNodes and continue. " + ex, ex);
+          DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
+            + ", add to deadNodes and continue. " + ex, ex);
           // Put chosen node into dead list, continue
           addToDeadNodes(chosenNode);
         }
@@ -696,7 +706,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
           "unreleased ByteBuffers allocated by read().  " +
           "Please release " + builder.toString() + ".");
     }
-    closeCurrentBlockReader();
+    closeCurrentBlockReaders();
     super.close();
   }
 
@@ -713,12 +723,22 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
    * Wraps different possible read implementations so that readBuffer can be
    * strategy-agnostic.
    */
-  private interface ReaderStrategy {
+  interface ReaderStrategy {
     public int doRead(BlockReader blockReader, int off, int len)
         throws ChecksumException, IOException;
+
+    /**
+     * Copy data from the src ByteBuffer into the read buffer.
+     * @param src The src buffer where the data is copied from
+     * @param offset Useful only when the ReadStrategy is based on a byte array.
+     *               Indicate the offset of the byte array for copy.
+     * @param length Useful only when the ReadStrategy is based on a byte array.
+     *               Indicate the length of the data to copy.
+     */
+    public int copyFrom(ByteBuffer src, int offset, int length);
   }
 
-  private void updateReadStatistics(ReadStatistics readStatistics, 
+  protected void updateReadStatistics(ReadStatistics readStatistics,
         int nRead, BlockReader blockReader) {
     if (nRead <= 0) return;
     synchronized(infoLock) {
@@ -749,12 +769,19 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
       updateReadStatistics(readStatistics, nRead, blockReader);
       return nRead;
     }
+
+    @Override
+    public int copyFrom(ByteBuffer src, int offset, int length) {
+      ByteBuffer writeSlice = src.duplicate();
+      writeSlice.get(buf, offset, length);
+      return length;
+    }
   }
 
   /**
    * Used to read bytes into a user-supplied ByteBuffer
    */
-  private class ByteBufferStrategy implements ReaderStrategy {
+  protected class ByteBufferStrategy implements ReaderStrategy {
     final ByteBuffer buf;
     ByteBufferStrategy(ByteBuffer buf) {
       this.buf = buf;
@@ -770,6 +797,9 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
         int ret = blockReader.read(buf);
         success = true;
         updateReadStatistics(readStatistics, ret, blockReader);
+        if (ret == 0) {
+          DFSClient.LOG.warn("zero");
+        }
         return ret;
       } finally {
         if (!success) {
@@ -779,6 +809,15 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
         }
       } 
     }
+
+    @Override
+    public int copyFrom(ByteBuffer src, int offset, int length) {
+      ByteBuffer writeSlice = src.duplicate();
+      int remaining = Math.min(buf.remaining(), writeSlice.remaining());
+      writeSlice.limit(writeSlice.position() + remaining);
+      buf.put(writeSlice);
+      return remaining;
+    }
   }
 
   /* This is a used by regular read() and handles ChecksumExceptions.
@@ -837,7 +876,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     }
   }
 
-  private synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
+  protected synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
     dfsClient.checkOpen();
     if (closed.get()) {
       throw new IOException("Stream closed");
@@ -926,7 +965,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   /**
    * Add corrupted block replica into map.
    */
-  private void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node, 
+  protected void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node,
       Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
     Set<DatanodeInfo> dnSet = null;
     if((corruptedBlockMap.containsKey(blk))) {
@@ -985,8 +1024,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
         } catch (InterruptedException iex) {
         }
         deadNodes.clear(); //2nd option is to remove only nodes[blockId]
-        openInfo();
-        block = getBlockAt(block.getStartOffset());
+        openInfo(true);
+        block = refreshLocatedBlock(block);
         failures++;
       }
     }
@@ -998,7 +1037,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
    * @param ignoredNodes Do not choose nodes in this array (may be null)
    * @return The DNAddrPair of the best node. Null if no node can be chosen.
    */
-  private DNAddrPair getBestNodeDNAddrPair(LocatedBlock block,
+  protected DNAddrPair getBestNodeDNAddrPair(LocatedBlock block,
       Collection<DatanodeInfo> ignoredNodes) {
     DatanodeInfo[] nodes = block.getLocations();
     StorageType[] storageTypes = block.getStorageTypes();
@@ -1058,15 +1097,15 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     return errMsgr.toString();
   }
 
-  private void fetchBlockByteRange(long blockStartOffset, long start, long end,
+  protected void fetchBlockByteRange(LocatedBlock block, long start, long end,
       byte[] buf, int offset,
       Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
       throws IOException {
-    LocatedBlock block = getBlockAt(blockStartOffset);
+    block = refreshLocatedBlock(block);
     while (true) {
       DNAddrPair addressPair = chooseDataNode(block, null);
       try {
-        actualGetFromOneDataNode(addressPair, blockStartOffset, start, end,
+        actualGetFromOneDataNode(addressPair, block, start, end,
             buf, offset, corruptedBlockMap);
         return;
       } catch (IOException e) {
@@ -1077,7 +1116,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   }
 
   private Callable<ByteBuffer> getFromOneDataNode(final DNAddrPair datanode,
-      final long blockStartOffset, final long start, final long end,
+      final LocatedBlock block, final long start, final long end,
       final ByteBuffer bb,
       final Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap,
       final int hedgedReadId) {
@@ -1090,7 +1129,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
         TraceScope scope =
             Trace.startSpan("hedgedRead" + hedgedReadId, parentSpan);
         try {
-          actualGetFromOneDataNode(datanode, blockStartOffset, start, end, buf,
+          actualGetFromOneDataNode(datanode, block, start, end, buf,
               offset, corruptedBlockMap);
           return bb;
         } finally {
@@ -1100,31 +1139,60 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     };
   }
 
+  /**
+   * Used when reading contiguous blocks
+   */
   private void actualGetFromOneDataNode(final DNAddrPair datanode,
-      long blockStartOffset, final long start, final long end, byte[] buf,
+      LocatedBlock block, final long start, final long end, byte[] buf,
       int offset, Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
       throws IOException {
+    final int length = (int) (end - start + 1);
+    actualGetFromOneDataNode(datanode, block, start, end, buf,
+        new int[]{offset}, new int[]{length}, corruptedBlockMap);
+  }
+
+  /**
+   * Read data from one DataNode.
+   * @param datanode the datanode from which to read data
+   * @param block the located block containing the requested data
+   * @param startInBlk the startInBlk offset of the block
+   * @param endInBlk the endInBlk offset of the block
+   * @param buf the given byte array into which the data is read
+   * @param offsets the data may be read into multiple segments of the buf
+   *                (when reading a striped block). this array indicates the
+   *                offset of each buf segment.
+   * @param lengths the length of each buf segment
+   * @param corruptedBlockMap map recording list of datanodes with corrupted
+   *                          block replica
+   */
+  void actualGetFromOneDataNode(final DNAddrPair datanode,
+      LocatedBlock block, final long startInBlk, final long endInBlk,
+      byte[] buf, int[] offsets, int[] lengths,
+      Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+      throws IOException {
     DFSClientFaultInjector.get().startFetchFromDatanode();
     int refetchToken = 1; // only need to get a new access token once
     int refetchEncryptionKey = 1; // only need to get a new encryption key once
+    final int len = (int) (endInBlk - startInBlk + 1);
+    checkReadPortions(offsets, lengths, len);
 
     while (true) {
       // cached block locations may have been updated by chooseDataNode()
       // or fetchBlockAt(). Always get the latest list of locations at the
       // start of the loop.
-      LocatedBlock block = getBlockAt(blockStartOffset);
+      block = refreshLocatedBlock(block);
       BlockReader reader = null;
       try {
         DFSClientFaultInjector.get().fetchFromDatanodeException();
-        int len = (int) (end - start + 1);
-        reader = getBlockReader(block, start, len, datanode.addr,
+        reader = getBlockReader(block, startInBlk, len, datanode.addr,
             datanode.storageType, datanode.info);
-        int nread = reader.readAll(buf, offset, len);
-        updateReadStatistics(readStatistics, nread, reader);
-
-        if (nread != len) {
-          throw new IOException("truncated return from reader.read(): " +
-                                "excpected " + len + ", got " + nread);
+        for (int i = 0; i < offsets.length; i++) {
+          int nread = reader.readAll(buf, offsets[i], lengths[i]);
+          updateReadStatistics(readStatistics, nread, reader);
+          if (nread != lengths[i]) {
+            throw new IOException("truncated return from reader.read(): " +
+                "excpected " + lengths[i] + ", got " + nread);
+          }
         }
         DFSClientFaultInjector.get().readFromDatanodeDelay();
         return;
@@ -1169,11 +1237,40 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   }
 
   /**
-   * Like {@link #fetchBlockByteRange} except we start up a second, parallel,
+   * Refresh cached block locations.
+   * @param block The currently cached block locations
+   * @return Refreshed block locations
+   * @throws IOException
+   */
+  protected LocatedBlock refreshLocatedBlock(LocatedBlock block)
+      throws IOException {
+    return getBlockAt(block.getStartOffset());
+  }
+
+  /**
+   * This method verifies that the read portions are valid and do not overlap
+   * with each other.
+   */
+  private void checkReadPortions(int[] offsets, int[] lengths, int totalLen) {
+    Preconditions.checkArgument(offsets.length == lengths.length && offsets.length > 0);
+    int sum = 0;
+    for (int i = 0; i < lengths.length; i++) {
+      if (i > 0) {
+        int gap = offsets[i] - offsets[i - 1];
+        // make sure read portions do not overlap with each other
+        Preconditions.checkArgument(gap >= lengths[i - 1]);
+      }
+      sum += lengths[i];
+    }
+    Preconditions.checkArgument(sum == totalLen);
+  }
+
+  /**
+   * Like {@link #fetchBlockByteRange}except we start up a second, parallel,
    * 'hedged' read if the first read is taking longer than configured amount of
    * time. We then wait on which ever read returns first.
    */
-  private void hedgedFetchBlockByteRange(long blockStartOffset, long start,
+  private void hedgedFetchBlockByteRange(LocatedBlock block, long start,
       long end, byte[] buf, int offset,
       Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
       throws IOException {
@@ -1186,7 +1283,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     ByteBuffer bb = null;
     int len = (int) (end - start + 1);
     int hedgedReadId = 0;
-    LocatedBlock block = getBlockAt(blockStartOffset);
+    block = refreshLocatedBlock(block);
     while (true) {
       // see HDFS-6591, this metric is used to verify/catch unnecessary loops
       hedgedReadOpsLoopNumForTesting++;
@@ -1198,7 +1295,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
         chosenNode = chooseDataNode(block, ignored);
         bb = ByteBuffer.wrap(buf, offset, len);
         Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
-            chosenNode, block.getStartOffset(), start, end, bb,
+            chosenNode, block, start, end, bb,
             corruptedBlockMap, hedgedReadId++);
         Future<ByteBuffer> firstRequest = hedgedService
             .submit(getFromDataNodeCallable);
@@ -1235,7 +1332,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
           }
           bb = ByteBuffer.allocate(len);
           Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
-              chosenNode, block.getStartOffset(), start, end, bb,
+              chosenNode, block, start, end, bb,
               corruptedBlockMap, hedgedReadId++);
           Future<ByteBuffer> oneMoreRequest = hedgedService
               .submit(getFromDataNodeCallable);
@@ -1319,7 +1416,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
    * @return true if block access token has expired or invalid and it should be
    *         refetched
    */
-  private static boolean tokenRefetchNeeded(IOException ex,
+  protected static boolean tokenRefetchNeeded(IOException ex,
       InetSocketAddress targetAddr) {
     /*
      * Get a new access token and retry. Retry is needed in 2 cases. 1)
@@ -1389,13 +1486,11 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
       long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
       try {
         if (dfsClient.isHedgedReadsEnabled()) {
-          hedgedFetchBlockByteRange(blk.getStartOffset(), targetStart,
-              targetStart + bytesToRead - 1, buffer, offset,
-              corruptedBlockMap);
+          hedgedFetchBlockByteRange(blk, targetStart,
+              targetStart + bytesToRead - 1, buffer, offset, corruptedBlockMap);
         } else {
-          fetchBlockByteRange(blk.getStartOffset(), targetStart,
-              targetStart + bytesToRead - 1, buffer, offset,
-              corruptedBlockMap);
+          fetchBlockByteRange(blk, targetStart, targetStart + bytesToRead - 1,
+              buffer, offset, corruptedBlockMap);
         }
       } finally {
         // Check and report if any block replicas are corrupted.
@@ -1427,7 +1522,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
    * @param corruptedBlockMap map of corrupted blocks
    * @param dataNodeCount number of data nodes who contains the block replicas
    */
-  private void reportCheckSumFailure(
+  protected void reportCheckSumFailure(
       Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap, 
       int dataNodeCount) {
     if (corruptedBlockMap.isEmpty()) {
@@ -1556,7 +1651,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   /**
    */
   @Override
-  public synchronized long getPos() throws IOException {
+  public synchronized long getPos() {
     return pos;
   }
 
@@ -1590,7 +1685,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
   }
 
   /** Utility class to encapsulate data node info and its address. */
-  private static final class DNAddrPair {
+  static final class DNAddrPair {
     final DatanodeInfo info;
     final InetSocketAddress addr;
     final StorageType storageType;
@@ -1627,7 +1722,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
     }
   }
 
-  private void closeCurrentBlockReader() {
+  protected void closeCurrentBlockReaders() {
     if (blockReader == null) return;
     // Close the current block reader so that the new caching settings can 
     // take effect immediately.
@@ -1647,7 +1742,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
       this.cachingStrategy =
           new CachingStrategy.Builder(this.cachingStrategy).setReadahead(readahead).build();
     }
-    closeCurrentBlockReader();
+    closeCurrentBlockReaders();
   }
 
   @Override
@@ -1657,7 +1752,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
       this.cachingStrategy =
           new CachingStrategy.Builder(this.cachingStrategy).setDropBehind(dropBehind).build();
     }
-    closeCurrentBlockReader();
+    closeCurrentBlockReaders();
   }
 
   /**
@@ -1815,6 +1910,6 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
 
   @Override
   public synchronized void unbuffer() {
-    closeCurrentBlockReader();
+    closeCurrentBlockReaders();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/825dbb82/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 68cc155..43e0eb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -343,7 +343,7 @@ public class TestDFSClientRetries {
       // we're starting a new operation on the user level.
       doAnswer(new FailNTimesAnswer(preSpyNN, maxBlockAcquires))
         .when(spyNN).getBlockLocations(anyString(), anyLong(), anyLong());
-      is.openInfo();
+      is.openInfo(true);
       // Seek to beginning forces a reopen of the BlockReader - otherwise it'll
       // just keep reading on the existing stream and the fact that we've poisoned
       // the block info won't do anything.


[33/48] hadoop git commit: MAPREDUCE-6376. Add avro binary support for jhist files. Contributed by Ray Chiang

Posted by zj...@apache.org.
MAPREDUCE-6376. Add avro binary support for jhist files. Contributed by Ray Chiang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23ac91c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23ac91c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23ac91c9

Branch: refs/heads/YARN-2928
Commit: 23ac91c90964e0af9d1f6b9a4b6de871b9be9e72
Parents: 14bfd5d
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 16:00:03 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 ++
 .../jobhistory/JobHistoryEventHandler.java      | 19 +++++++++++--
 .../hadoop/mapreduce/jobhistory/TestEvents.java |  3 +-
 .../mapreduce/v2/jobhistory/JHAdminConfig.java  |  7 +++++
 .../mapreduce/jobhistory/EventReader.java       | 12 ++++----
 .../mapreduce/jobhistory/EventWriter.java       | 29 ++++++++++++++++----
 .../src/main/resources/mapred-default.xml       |  9 ++++++
 7 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d170b55..49a873d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -396,6 +396,9 @@ Release 2.8.0 - UNRELEASED
 
   OPTIMIZATIONS
 
+    MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via
+    jlowe)
+
   BUG FIXES
 
     MAPREDUCE-6314. TestPipeApplication fails on trunk.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 5208874..a8f10d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -108,7 +108,8 @@ public class JobHistoryEventHandler extends AbstractService
 
   private int numUnflushedCompletionEvents = 0;
   private boolean isTimerActive;
-
+  private EventWriter.WriteMode jhistMode =
+      EventWriter.WriteMode.JSON;
 
   protected BlockingQueue<JobHistoryEvent> eventQueue =
     new LinkedBlockingQueue<JobHistoryEvent>();
@@ -282,6 +283,20 @@ public class JobHistoryEventHandler extends AbstractService
       LOG.info("Emitting job history data to the timeline server is not enabled");
     }
 
+    // Flag for setting
+    String jhistFormat = conf.get(JHAdminConfig.MR_HS_JHIST_FORMAT,
+        JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT);
+    if (jhistFormat.equals("json")) {
+      jhistMode = EventWriter.WriteMode.JSON;
+    } else if (jhistFormat.equals("binary")) {
+      jhistMode = EventWriter.WriteMode.BINARY;
+    } else {
+      LOG.warn("Unrecognized value '" + jhistFormat + "' for property " +
+          JHAdminConfig.MR_HS_JHIST_FORMAT + ".  Valid values are " +
+          "'json' or 'binary'.  Falling back to default value '" +
+          JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT + "'.");
+    }
+
     super.serviceInit(conf);
   }
 
@@ -457,7 +472,7 @@ public class JobHistoryEventHandler extends AbstractService
   protected EventWriter createEventWriter(Path historyFilePath)
       throws IOException {
     FSDataOutputStream out = stagingDirFS.create(historyFilePath, true);
-    return new EventWriter(out);
+    return new EventWriter(out, this.jhistMode);
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 27fcb33..e7d5006 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -191,7 +191,8 @@ public class TestEvents {
     ByteArrayOutputStream output = new ByteArrayOutputStream();
     FSDataOutputStream fsOutput = new FSDataOutputStream(output,
         new FileSystem.Statistics("scheme"));
-    EventWriter writer = new EventWriter(fsOutput);
+    EventWriter writer = new EventWriter(fsOutput,
+        EventWriter.WriteMode.JSON);
     writer.write(getJobPriorityChangedEvent());
     writer.write(getJobStatusChangedEvent());
     writer.write(getTaskUpdatedEvent());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index a97c2ca..86dfad3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -221,4 +221,11 @@ public class JHAdminConfig {
       + "jobname.limit";
   public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50;
 
+  /**
+   * Settings for .jhist file format.
+   */
+  public static final String MR_HS_JHIST_FORMAT =
+      MR_HISTORY_PREFIX + "jhist.format";
+  public static final String DEFAULT_MR_HS_JHIST_FORMAT =
+      "json";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
index e08a929..9898c2d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
@@ -66,16 +66,18 @@ public class EventReader implements Closeable {
   public EventReader(DataInputStream in) throws IOException {
     this.in = in;
     this.version = in.readLine();
-    
-    if (!EventWriter.VERSION.equals(version)) {
-      throw new IOException("Incompatible event log version: "+version);
-    }
 
     Schema myschema = new SpecificData(Event.class.getClassLoader()).getSchema(Event.class);
     Schema.Parser parser = new Schema.Parser();
     this.schema = parser.parse(in.readLine());
     this.reader = new SpecificDatumReader(schema, myschema);
-    this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
+    if (EventWriter.VERSION.equals(version)) {
+      this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
+    } else if (EventWriter.VERSION_BINARY.equals(version)) {
+      this.decoder = DecoderFactory.get().binaryDecoder(in, null);
+    } else {
+      throw new IOException("Incompatible event log version: " + version);
+    }
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
index a548dfe..29489a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
@@ -43,20 +43,37 @@ import org.apache.hadoop.mapreduce.Counters;
  */
 class EventWriter {
   static final String VERSION = "Avro-Json";
+  static final String VERSION_BINARY = "Avro-Binary";
 
   private FSDataOutputStream out;
   private DatumWriter<Event> writer =
     new SpecificDatumWriter<Event>(Event.class);
   private Encoder encoder;
   private static final Log LOG = LogFactory.getLog(EventWriter.class);
-  
-  EventWriter(FSDataOutputStream out) throws IOException {
+  public enum WriteMode { JSON, BINARY }
+  private final WriteMode writeMode;
+  private final boolean jsonOutput;  // Cache value while we have 2 modes
+
+  EventWriter(FSDataOutputStream out, WriteMode mode) throws IOException {
     this.out = out;
-    out.writeBytes(VERSION);
+    this.writeMode = mode;
+    if (this.writeMode==WriteMode.JSON) {
+      this.jsonOutput = true;
+      out.writeBytes(VERSION);
+    } else if (this.writeMode==WriteMode.BINARY) {
+      this.jsonOutput = false;
+      out.writeBytes(VERSION_BINARY);
+    } else {
+      throw new IOException("Unknown mode: " + mode);
+    }
     out.writeBytes("\n");
     out.writeBytes(Event.SCHEMA$.toString());
     out.writeBytes("\n");
-    this.encoder =  EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
+    if (!this.jsonOutput) {
+      this.encoder = EncoderFactory.get().binaryEncoder(out, null);
+    } else {
+      this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
+    }
   }
   
   synchronized void write(HistoryEvent event) throws IOException { 
@@ -65,7 +82,9 @@ class EventWriter {
     wrapper.setEvent(event.getDatum());
     writer.write(wrapper, encoder);
     encoder.flush();
-    out.writeBytes("\n");
+    if (this.jsonOutput) {
+      out.writeBytes("\n");
+    }
   }
   
   void flush() throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ac91c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index c89a566..8d3450b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1721,6 +1721,15 @@
 </property>
 
 <property>
+  <description>
+  File format the AM will use when generating the .jhist file.  Valid
+  values are "json" for text output and "binary" for faster parsing.
+  </description>
+  <name>mapreduce.jobhistory.jhist.format</name>
+  <value>json</value>
+</property>
+
+<property>
   <name>mapreduce.job.heap.memory-mb.ratio</name>
   <value>0.8</value>
   <description>The ratio of heap-size to container-size. If no -Xmx is


[11/48] hadoop git commit: Revert "HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)"

Posted by zj...@apache.org.
Revert "HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix  FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)"

This reverts commit 3dfa8161f9412bcb040f3c29c471344f25f24337.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/405b0982
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/405b0982
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/405b0982

Branch: refs/heads/YARN-2928
Commit: 405b0982b600a5c7c44ebf8c6644db15a43e2091
Parents: d963454
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 30 11:54:12 2015 +0100
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt    |  4 ----
 .../main/java/org/apache/hadoop/fs/FileSystem.java | 17 +----------------
 .../src/site/markdown/filesystem/filesystem.md     |  4 ----
 .../hadoop/fs/FileSystemContractBaseTest.java      | 11 +++--------
 4 files changed, 4 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2cf9509..c010ff1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -664,10 +664,6 @@ Release 2.8.0 - UNRELEASED
 
     HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
     occurs (Jason Lowe via jeagles)
-
-    HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix
-    FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)
-
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c98074a..c73caf7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1498,9 +1498,7 @@ public abstract class FileSystem extends Configured implements Closeable {
   /**
    * List the statuses of the files/directories in the given path if the path is
    * a directory.
-   * <p>
-   * Does not guarantee to return the List of files/directories status in a
-   * sorted order.
+   * 
    * @param f given path
    * @return the statuses of the files/directories in the given patch
    * @throws FileNotFoundException when the path does not exist;
@@ -1542,9 +1540,6 @@ public abstract class FileSystem extends Configured implements Closeable {
   /**
    * Filter files/directories in the given path using the user-supplied path
    * filter.
-   * <p>
-   * Does not guarantee to return the List of files/directories status in a
-   * sorted order.
    * 
    * @param f
    *          a path name
@@ -1565,9 +1560,6 @@ public abstract class FileSystem extends Configured implements Closeable {
   /**
    * Filter files/directories in the given list of paths using default
    * path filter.
-   * <p>
-   * Does not guarantee to return the List of files/directories status in a
-   * sorted order.
    * 
    * @param files
    *          a list of paths
@@ -1584,9 +1576,6 @@ public abstract class FileSystem extends Configured implements Closeable {
   /**
    * Filter files/directories in the given list of paths using user-supplied
    * path filter.
-   * <p>
-   * Does not guarantee to return the List of files/directories status in a
-   * sorted order.
    * 
    * @param files
    *          a list of paths
@@ -1747,8 +1736,6 @@ public abstract class FileSystem extends Configured implements Closeable {
    * while consuming the entries. Each file system implementation should
    * override this method and provide a more efficient implementation, if
    * possible. 
-   * Does not guarantee to return the iterator that traverses statuses
-   * of the files in a sorted order.
    *
    * @param p target path
    * @return remote iterator
@@ -1776,8 +1763,6 @@ public abstract class FileSystem extends Configured implements Closeable {
 
   /**
    * List the statuses and block locations of the files in the given path.
-   * Does not guarantee to return the iterator that traverses statuses
-   * of the files in a sorted order.
    * 
    * If the path is a directory, 
    *   if recursive is false, returns files in the directory;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index f323374..84e3755 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -183,10 +183,6 @@ to the same path:
     forall fs in listStatus(Path) :
       fs == getFileStatus(fs.path)
 
-**Ordering of results**: there is no guarantee of ordering of the listed entries.
-While HDFS currently returns an alphanumerically sorted list, neither the Posix `readdir()`
-nor Java's `File.listFiles()` API calls define any ordering of returned values. Applications
-which require a uniform sort order on the results must perform the sorting themselves.
 
 ### Atomicity and Consistency
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/405b0982/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 495af00..2ca81e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.util.ArrayList;
 
 import junit.framework.TestCase;
 
@@ -225,13 +224,9 @@ public abstract class FileSystemContractBaseTest extends TestCase {
 
     paths = fs.listStatus(path("/test/hadoop"));
     assertEquals(3, paths.length);
-    ArrayList<String> list = new ArrayList<String>();
-    for (FileStatus fileState : paths) {
-      list.add(fileState.getPath().toString());
-    }
-    assertTrue(list.contains(path("/test/hadoop/a")));
-    assertTrue(list.contains(path("/test/hadoop/b")));
-    assertTrue(list.contains(path("/test/hadoop/c")));
+    assertEquals(path("/test/hadoop/a"), paths[0].getPath());
+    assertEquals(path("/test/hadoop/b"), paths[1].getPath());
+    assertEquals(path("/test/hadoop/c"), paths[2].getPath());
 
     paths = fs.listStatus(path("/test/hadoop/a"));
     assertEquals(0, paths.length);


[36/48] hadoop git commit: YARN-3508. Prevent processing preemption events on the main RM dispatcher. (Varun Saxena via wangda)

Posted by zj...@apache.org.
YARN-3508. Prevent processing preemption events on the main RM dispatcher. (Varun Saxena via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5192ca9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5192ca9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5192ca9

Branch: refs/heads/YARN-2928
Commit: d5192ca993d8c76fe0b51c3a91967fb0c2bbf0a1
Parents: 2820f1f
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Jul 1 17:32:22 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../server/resourcemanager/ResourceManager.java | 34 ---------
 .../monitor/SchedulingEditPolicy.java           |  6 +-
 .../monitor/SchedulingMonitor.java              |  3 +-
 .../ProportionalCapacityPreemptionPolicy.java   | 42 ++++++-----
 .../scheduler/ContainerPreemptEvent.java        |  8 +-
 .../scheduler/ContainerPreemptEventType.java    | 26 -------
 .../scheduler/capacity/CapacityScheduler.java   | 24 ++++++
 .../scheduler/event/SchedulerEventType.java     |  7 +-
 .../resourcemanager/TestRMDispatcher.java       | 79 ++++++++++++++++++++
 ...estProportionalCapacityPreemptionPolicy.java | 19 +++--
 ...pacityPreemptionPolicyForNodePartitions.java | 12 ++-
 12 files changed, 161 insertions(+), 102 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c4c7fa9..3e42e61 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -704,6 +704,9 @@ Release 2.7.2 - UNRELEASED
     YARN-3793. Several NPEs when deleting local files on NM recovery (Varun
     Saxena via jlowe)
 
+    YARN-3508. Prevent processing preemption events on the main RM dispatcher. 
+    (Varun Saxena via wangda)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 3302320..b235058 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -76,7 +76,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
@@ -624,9 +623,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
             YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
             SchedulingEditPolicy.class);
         if (policies.size() > 0) {
-          rmDispatcher.register(ContainerPreemptEventType.class,
-              new RMContainerPreemptEventDispatcher(
-                  (PreemptableResourceScheduler) scheduler));
           for (SchedulingEditPolicy policy : policies) {
             LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
             // periodically check whether we need to take action to guarantee
@@ -797,36 +793,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
   }
 
   @Private
-  public static final class
-    RMContainerPreemptEventDispatcher
-      implements EventHandler<ContainerPreemptEvent> {
-
-    private final PreemptableResourceScheduler scheduler;
-
-    public RMContainerPreemptEventDispatcher(
-        PreemptableResourceScheduler scheduler) {
-      this.scheduler = scheduler;
-    }
-
-    @Override
-    public void handle(ContainerPreemptEvent event) {
-      ApplicationAttemptId aid = event.getAppId();
-      RMContainer container = event.getContainer();
-      switch (event.getType()) {
-      case DROP_RESERVATION:
-        scheduler.dropContainerReservation(container);
-        break;
-      case PREEMPT_CONTAINER:
-        scheduler.preemptContainer(aid, container);
-        break;
-      case KILL_CONTAINER:
-        scheduler.killContainer(container);
-        break;
-      }
-    }
-  }
-
-  @Private
   public static final class ApplicationAttemptEventDispatcher implements
       EventHandler<RMAppAttemptEvent> {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
index 1ebc19f..0d587d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
@@ -18,14 +18,12 @@
 package org.apache.hadoop.yarn.server.resourcemanager.monitor;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
 
 public interface SchedulingEditPolicy {
 
-  public void init(Configuration config,
-      EventHandler<ContainerPreemptEvent> dispatcher,
+  public void init(Configuration config, RMContext context,
       PreemptableResourceScheduler scheduler);
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
index 1682f7d..d4c129b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
@@ -54,9 +54,8 @@ public class SchedulingMonitor extends AbstractService {
     return scheduleEditPolicy;
   }
 
-  @SuppressWarnings("unchecked")
   public void serviceInit(Configuration conf) throws Exception {
-    scheduleEditPolicy.init(conf, rmContext.getDispatcher().getEventHandler(),
+    scheduleEditPolicy.init(conf, rmContext,
         (PreemptableResourceScheduler) rmContext.getScheduler());
     this.monitorInterval = scheduleEditPolicy.getMonitoringInterval();
     super.serviceInit(conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1f47b5f..5a20a6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -38,13 +38,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -52,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.SystemClock;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -118,8 +118,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
   public static final String NATURAL_TERMINATION_FACTOR =
       "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor";
 
-  // the dispatcher to send preempt and kill events
-  public EventHandler<ContainerPreemptEvent> dispatcher;
+  private RMContext rmContext;
 
   private final Clock clock;
   private double maxIgnoredOverCapacity;
@@ -141,20 +140,17 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
   }
 
   public ProportionalCapacityPreemptionPolicy(Configuration config,
-      EventHandler<ContainerPreemptEvent> dispatcher,
-      CapacityScheduler scheduler) {
-    this(config, dispatcher, scheduler, new SystemClock());
+      RMContext context, CapacityScheduler scheduler) {
+    this(config, context, scheduler, new SystemClock());
   }
 
   public ProportionalCapacityPreemptionPolicy(Configuration config,
-      EventHandler<ContainerPreemptEvent> dispatcher,
-      CapacityScheduler scheduler, Clock clock) {
-    init(config, dispatcher, scheduler);
+      RMContext context, CapacityScheduler scheduler, Clock clock) {
+    init(config, context, scheduler);
     this.clock = clock;
   }
 
-  public void init(Configuration config,
-      EventHandler<ContainerPreemptEvent> disp,
+  public void init(Configuration config, RMContext context,
       PreemptableResourceScheduler sched) {
     LOG.info("Preemption monitor:" + this.getClass().getCanonicalName());
     assert null == scheduler : "Unexpected duplicate call to init";
@@ -163,7 +159,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
           sched.getClass().getCanonicalName() + " not instance of " +
           CapacityScheduler.class.getCanonicalName());
     }
-    dispatcher = disp;
+    rmContext = context;
     scheduler = (CapacityScheduler) sched;
     maxIgnoredOverCapacity = config.getDouble(MAX_IGNORED_OVER_CAPACITY, 0.1);
     naturalTerminationFactor =
@@ -196,6 +192,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
    * @param root the root of the CapacityScheduler queue hierarchy
    * @param clusterResources the total amount of resources in the cluster
    */
+  @SuppressWarnings("unchecked")
   private void containerBasedPreemptOrKill(CSQueue root,
       Resource clusterResources) {
     // All partitions to look at
@@ -248,8 +245,9 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
     // preempt (or kill) the selected containers
     for (Map.Entry<ApplicationAttemptId,Set<RMContainer>> e
          : toPreempt.entrySet()) {
+      ApplicationAttemptId appAttemptId = e.getKey();
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Send to scheduler: in app=" + e.getKey()
+        LOG.debug("Send to scheduler: in app=" + appAttemptId
             + " #containers-to-be-preempted=" + e.getValue().size());
       }
       for (RMContainer container : e.getValue()) {
@@ -257,13 +255,15 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
         if (preempted.get(container) != null &&
             preempted.get(container) + maxWaitTime < clock.getTime()) {
           // kill it
-          dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
-                ContainerPreemptEventType.KILL_CONTAINER));
+          rmContext.getDispatcher().getEventHandler().handle(
+              new ContainerPreemptEvent(appAttemptId, container,
+                  SchedulerEventType.KILL_CONTAINER));
           preempted.remove(container);
         } else {
           //otherwise just send preemption events
-          dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
-                ContainerPreemptEventType.PREEMPT_CONTAINER));
+          rmContext.getDispatcher().getEventHandler().handle(
+              new ContainerPreemptEvent(appAttemptId, container,
+                  SchedulerEventType.PREEMPT_CONTAINER));
           if (preempted.get(container) == null) {
             preempted.put(container, clock.getTime());
           }
@@ -735,6 +735,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
    * Given a target preemption for a specific application, select containers
    * to preempt (after unreserving all reservation for that app).
    */
+  @SuppressWarnings("unchecked")
   private void preemptFrom(FiCaSchedulerApp app,
       Resource clusterResource, Map<String, Resource> resToObtainByPartition,
       List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
@@ -758,8 +759,9 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
           clusterResource, preemptMap);
 
       if (!observeOnly) {
-        dispatcher.handle(new ContainerPreemptEvent(appId, c,
-            ContainerPreemptEventType.DROP_RESERVATION));
+        rmContext.getDispatcher().getEventHandler().handle(
+            new ContainerPreemptEvent(
+                appId, c, SchedulerEventType.DROP_RESERVATION));
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
index 8eba48d..7ab2758 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEvent.java
@@ -19,20 +19,20 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.event.AbstractEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 
 /**
  * Simple event class used to communicate containers unreservations, preemption, killing
  */
-public class ContainerPreemptEvent
-    extends AbstractEvent<ContainerPreemptEventType> {
+public class ContainerPreemptEvent extends SchedulerEvent {
 
   private final ApplicationAttemptId aid;
   private final RMContainer container;
 
   public ContainerPreemptEvent(ApplicationAttemptId aid, RMContainer container,
-      ContainerPreemptEventType type) {
+      SchedulerEventType type) {
     super(type);
     this.aid = aid;
     this.container = container;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java
deleted file mode 100644
index a70a836..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerPreemptEventType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-
-public enum ContainerPreemptEventType {
-
-  DROP_RESERVATION,
-  PREEMPT_CONTAINER,
-  KILL_CONTAINER
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index f1d0f9c..141aa7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -1346,6 +1347,29 @@ public class CapacityScheduler extends
           RMContainerEventType.EXPIRE);
     }
     break;
+    case DROP_RESERVATION:
+    {
+      ContainerPreemptEvent dropReservationEvent = (ContainerPreemptEvent)event;
+      RMContainer container = dropReservationEvent.getContainer();
+      dropContainerReservation(container);
+    }
+    break;
+    case PREEMPT_CONTAINER:
+    {
+      ContainerPreemptEvent preemptContainerEvent =
+          (ContainerPreemptEvent)event;
+      ApplicationAttemptId aid = preemptContainerEvent.getAppId();
+      RMContainer containerToBePreempted = preemptContainerEvent.getContainer();
+      preemptContainer(aid, containerToBePreempted);
+    }
+    break;
+    case KILL_CONTAINER:
+    {
+      ContainerPreemptEvent killContainerEvent = (ContainerPreemptEvent)event;
+      RMContainer containerToBeKilled = killContainerEvent.getContainer();
+      killContainer(containerToBeKilled);
+    }
+    break;
     default:
       LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!");
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 13aecb3..9de935b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -36,5 +36,10 @@ public enum SchedulerEventType {
   APP_ATTEMPT_REMOVED,
 
   // Source: ContainerAllocationExpirer
-  CONTAINER_EXPIRED
+  CONTAINER_EXPIRED,
+
+  // Source: SchedulingEditPolicy
+  DROP_RESERVATION,
+  PREEMPT_CONTAINER,
+  KILL_CONTAINER
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java
new file mode 100644
index 0000000..db7c96a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMDispatcher.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.SchedulerEventDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestRMDispatcher {
+
+  @SuppressWarnings("unchecked")
+  @Test(timeout=10000)
+  public void testSchedulerEventDispatcherForPreemptionEvents() {
+    AsyncDispatcher rmDispatcher = new AsyncDispatcher();
+    CapacityScheduler sched = spy(new CapacityScheduler());
+    YarnConfiguration conf = new YarnConfiguration();
+    SchedulerEventDispatcher schedulerDispatcher =
+        new SchedulerEventDispatcher(sched);
+    rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
+    rmDispatcher.init(conf);
+    rmDispatcher.start();
+    schedulerDispatcher.init(conf);
+    schedulerDispatcher.start();
+    try {
+      ApplicationAttemptId appAttemptId = mock(ApplicationAttemptId.class);
+      RMContainer container = mock(RMContainer.class);
+      ContainerPreemptEvent event1 = new ContainerPreemptEvent(
+          appAttemptId, container, SchedulerEventType.DROP_RESERVATION);
+      rmDispatcher.getEventHandler().handle(event1);
+      ContainerPreemptEvent event2 = new ContainerPreemptEvent(
+           appAttemptId, container, SchedulerEventType.KILL_CONTAINER);
+      rmDispatcher.getEventHandler().handle(event2);
+      ContainerPreemptEvent event3 = new ContainerPreemptEvent(
+          appAttemptId, container, SchedulerEventType.PREEMPT_CONTAINER);
+      rmDispatcher.getEventHandler().handle(event3);
+      // Wait for events to be processed by scheduler dispatcher.
+      Thread.sleep(1000);
+      verify(sched, times(3)).handle(any(SchedulerEvent.class));
+      verify(sched).dropContainerReservation(container);
+      verify(sched).preemptContainer(appAttemptId, container);
+      verify(sched).killContainer(container);
+    } catch (InterruptedException e) {
+      Assert.fail();
+    } finally {
+      schedulerDispatcher.stop();
+      rmDispatcher.stop();
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 6c0ed6c..2c0c6d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -23,8 +23,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.Pro
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.OBSERVE_ONLY;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType.KILL_CONTAINER;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType.PREEMPT_CONTAINER;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.KILL_CONTAINER;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.PREEMPT_CONTAINER;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -66,7 +67,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
 import org.apache.hadoop.yarn.server.resourcemanager.resource.Priority;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueu
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -104,7 +106,7 @@ public class TestProportionalCapacityPreemptionPolicy {
   RMContext rmContext = null;
   RMNodeLabelsManager lm = null;
   CapacitySchedulerConfiguration schedConf = null;
-  EventHandler<ContainerPreemptEvent> mDisp = null;
+  EventHandler<SchedulerEvent> mDisp = null;
   ResourceCalculator rc = new DefaultResourceCalculator();
   Resource clusterResources = null;
   final ApplicationAttemptId appA = ApplicationAttemptId.newInstance(
@@ -164,6 +166,9 @@ public class TestProportionalCapacityPreemptionPolicy {
     when(mCS.getRMContext()).thenReturn(rmContext);
     when(rmContext.getNodeLabelManager()).thenReturn(lm);
     mDisp = mock(EventHandler.class);
+    Dispatcher disp = mock(Dispatcher.class);
+    when(rmContext.getDispatcher()).thenReturn(disp);
+    when(disp.getEventHandler()).thenReturn(mDisp);
     rand = new Random();
     long seed = rand.nextLong();
     System.out.println(name.getMethodName() + " SEED: " + seed);
@@ -866,12 +871,12 @@ public class TestProportionalCapacityPreemptionPolicy {
   static class IsPreemptionRequestFor
       extends ArgumentMatcher<ContainerPreemptEvent> {
     private final ApplicationAttemptId appAttId;
-    private final ContainerPreemptEventType type;
+    private final SchedulerEventType type;
     IsPreemptionRequestFor(ApplicationAttemptId appAttId) {
       this(appAttId, PREEMPT_CONTAINER);
     }
     IsPreemptionRequestFor(ApplicationAttemptId appAttId,
-        ContainerPreemptEventType type) {
+        SchedulerEventType type) {
       this.appAttId = appAttId;
       this.type = type;
     }
@@ -888,7 +893,7 @@ public class TestProportionalCapacityPreemptionPolicy {
 
   ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) {
     ProportionalCapacityPreemptionPolicy policy =
-      new ProportionalCapacityPreemptionPolicy(conf, mDisp, mCS, mClock);
+      new ProportionalCapacityPreemptionPolicy(conf, rmContext, mCS, mClock);
     ParentQueue mRoot = buildMockRootQueue(rand, qData);
     when(mCS.getRootQueue()).thenReturn(mRoot);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5192ca9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index e13320c..114769c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -50,13 +50,13 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueu
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -92,7 +93,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
   private Configuration conf = null;
   private CapacitySchedulerConfiguration csConf = null;
   private CapacityScheduler cs = null;
-  private EventHandler<ContainerPreemptEvent> mDisp = null;
+  private EventHandler<SchedulerEvent> mDisp = null;
   private ProportionalCapacityPreemptionPolicy policy = null;
   private Resource clusterResource = null;
 
@@ -125,11 +126,14 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
 
     rmContext = mock(RMContext.class);
     when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+    Dispatcher disp = mock(Dispatcher.class);
+    when(rmContext.getDispatcher()).thenReturn(disp);
+    when(disp.getEventHandler()).thenReturn(mDisp);
     csConf = new CapacitySchedulerConfiguration();
     when(cs.getConfiguration()).thenReturn(csConf);
     when(cs.getRMContext()).thenReturn(rmContext);
 
-    policy = new ProportionalCapacityPreemptionPolicy(conf, mDisp, cs, mClock);
+    policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, mClock);
     partitionToResource = new HashMap<>();
     nodeIdToSchedulerNodes = new HashMap<>();
     nameToCSQueues = new HashMap<>();
@@ -828,7 +832,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
     when(cs.getClusterResource()).thenReturn(clusterResource);
     mockApplications(appsConfig);
 
-    policy = new ProportionalCapacityPreemptionPolicy(conf, mDisp, cs, mClock);
+    policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, mClock);
   }
 
   private void mockContainers(String containersConfig, ApplicationAttemptId attemptId,


[02/48] hadoop git commit: HADOOP-12089. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. Contributed by Duo Xu.

Posted by zj...@apache.org.
HADOOP-12089. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. Contributed by Duo Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3aff2fdb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3aff2fdb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3aff2fdb

Branch: refs/heads/YARN-2928
Commit: 3aff2fdbe38fcb49e23908a1bfa83743fa604e70
Parents: 6f780d1
Author: cnauroth <cn...@apache.org>
Authored: Mon Jun 29 13:48:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                 | 3 +++
 .../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java  | 5 ++---
 2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aff2fdb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9008ead..a9b44e3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -893,6 +893,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12119. hadoop fs -expunge does not work for federated namespace
     (J.Andreina via vinayakumarb)
 
+    HADOOP-12089. StorageException complaining " no lease ID" when updating
+    FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aff2fdb/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 623645a..a567b33 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1500,7 +1500,7 @@ public class NativeAzureFileSystem extends FileSystem {
               createPermissionStatus(FsPermission.getDefault()));
         } else {
           if (!skipParentFolderLastModifidedTimeUpdate) {
-            store.updateFolderLastModifiedTime(parentKey, null);
+            updateParentFolderLastModifiedTime(key);
           }
         }
       }
@@ -1561,9 +1561,8 @@ public class NativeAzureFileSystem extends FileSystem {
       // Update parent directory last modified time
       Path parent = absolutePath.getParent();
       if (parent != null && parent.getParent() != null) { // not root
-        String parentKey = pathToKey(parent);
         if (!skipParentFolderLastModifidedTimeUpdate) {
-          store.updateFolderLastModifiedTime(parentKey, null);
+          updateParentFolderLastModifiedTime(key);
         }
       }
       instrumentation.directoryDeleted();


[20/48] hadoop git commit: HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)

Posted by zj...@apache.org.
HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c68da70b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c68da70b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c68da70b

Branch: refs/heads/YARN-2928
Commit: c68da70b70cc050234d05612ca75b442efe8db06
Parents: 405b098
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jun 30 11:31:26 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700

----------------------------------------------------------------------
 dev-support/test-patch.sh                       | 4 ++--
 hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c68da70b/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index cd91a5c..efcd614 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1521,8 +1521,8 @@ function check_reexec
 
   cd "${CWD}"
   mkdir -p "${PATCH_DIR}/dev-support-test"
-  cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test"
-  cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test"
+  (cd "${BINDIR}"; tar cpf - . ) \
+	| (cd  "${PATCH_DIR}/dev-support-test"; tar xpf - )
 
   big_console_header "exec'ing test-patch.sh now..."
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c68da70b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c010ff1..e332ea8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -232,6 +232,8 @@ Trunk (Unreleased)
 
     HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
 
+    HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)
+
   BUG FIXES
 
     HADOOP-11473. test-patch says "-1 overall" even when all checks are +1


[15/48] hadoop git commit: HDFS-8493. Consolidate truncate() related implementation in a single class. Contributed by Rakesh R.

Posted by zj...@apache.org.
HDFS-8493. Consolidate truncate() related implementation in a single class. Contributed by Rakesh R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e39b962
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e39b962
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e39b962

Branch: refs/heads/YARN-2928
Commit: 4e39b9622bd871c9e16cc22fdc85801fd8915e72
Parents: c51682b
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Jun 29 16:40:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../hdfs/server/namenode/FSDirTruncateOp.java   | 361 +++++++++++++++++++
 .../hdfs/server/namenode/FSDirectory.java       |  95 -----
 .../hdfs/server/namenode/FSEditLogLoader.java   |   6 +-
 .../hdfs/server/namenode/FSNamesystem.java      | 237 ++----------
 .../hdfs/server/namenode/TestFileTruncate.java  |   8 +-
 6 files changed, 402 insertions(+), 308 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 108a6c0..3535f90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and
     DatanodeStorageInfo. (Zhe Zhang via wang)
 
+    HDFS-8493. Consolidate truncate() related implementation in a single class.
+    (Rakesh R via wheat9)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
new file mode 100644
index 0000000..9fc9def
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -0,0 +1,361 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionContiguous;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.RecoverLeaseOp;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Helper class to perform truncate operation.
+ */
+final class FSDirTruncateOp {
+
+  /**
+   * Private constructor for preventing FSDirTruncateOp object creation.
+   * Static-only class.
+   */
+  private FSDirTruncateOp() {}
+
+  /**
+   * Truncate a file to a given size.
+   *
+   * @param fsn namespace
+   * @param srcArg path name
+   * @param newLength the target file size
+   * @param clientName client name
+   * @param clientMachine client machine info
+   * @param mtime modified time
+   * @param toRemoveBlocks to be removed blocks
+   * @param pc permission checker to check fs permission
+   * @return tuncate result
+   * @throws IOException
+   */
+  static TruncateResult truncate(final FSNamesystem fsn, final String srcArg,
+      final long newLength, final String clientName,
+      final String clientMachine, final long mtime,
+      final BlocksMapUpdateInfo toRemoveBlocks, final FSPermissionChecker pc)
+      throws IOException, UnresolvedLinkException {
+    assert fsn.hasWriteLock();
+
+    FSDirectory fsd = fsn.getFSDirectory();
+    byte[][] pathComponents = FSDirectory
+        .getPathComponentsForReservedPath(srcArg);
+    final String src;
+    final INodesInPath iip;
+    final boolean onBlockBoundary;
+    Block truncateBlock = null;
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, srcArg, pathComponents);
+      iip = fsd.getINodesInPath4Write(src, true);
+      if (fsn.isPermissionEnabled()) {
+        fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+      }
+      INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
+      final BlockStoragePolicy lpPolicy = fsn.getBlockManager()
+          .getStoragePolicy("LAZY_PERSIST");
+
+      if (lpPolicy != null && lpPolicy.getId() == file.getStoragePolicyID()) {
+        throw new UnsupportedOperationException(
+            "Cannot truncate lazy persist file " + src);
+      }
+
+      // Check if the file is already being truncated with the same length
+      final BlockInfo last = file.getLastBlock();
+      if (last != null && last.getBlockUCState()
+          == BlockUCState.UNDER_RECOVERY) {
+        final Block truncatedBlock = ((BlockInfoUnderConstruction) last)
+            .getTruncateBlock();
+        if (truncatedBlock != null) {
+          final long truncateLength = file.computeFileSize(false, false)
+              + truncatedBlock.getNumBytes();
+          if (newLength == truncateLength) {
+            return new TruncateResult(false, fsd.getAuditFileInfo(iip));
+          }
+        }
+      }
+
+      // Opening an existing file for truncate. May need lease recovery.
+      fsn.recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE, iip, src,
+          clientName, clientMachine, false);
+      // Truncate length check.
+      long oldLength = file.computeFileSize();
+      if (oldLength == newLength) {
+        return new TruncateResult(true, fsd.getAuditFileInfo(iip));
+      }
+      if (oldLength < newLength) {
+        throw new HadoopIllegalArgumentException(
+            "Cannot truncate to a larger file size. Current size: " + oldLength
+                + ", truncate size: " + newLength + ".");
+      }
+      // Perform INodeFile truncation.
+      final QuotaCounts delta = new QuotaCounts.Builder().build();
+      onBlockBoundary = unprotectedTruncate(fsn, iip, newLength,
+          toRemoveBlocks, mtime, delta);
+      if (!onBlockBoundary) {
+        // Open file for write, but don't log into edits
+        long lastBlockDelta = file.computeFileSize() - newLength;
+        assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
+        truncateBlock = prepareFileForTruncate(fsn, iip, clientName,
+            clientMachine, lastBlockDelta, null);
+      }
+
+      // update the quota: use the preferred block size for UC block
+      fsd.updateCountNoQuotaCheck(iip, iip.length() - 1, delta);
+    } finally {
+      fsd.writeUnlock();
+    }
+
+    fsn.getEditLog().logTruncate(src, clientName, clientMachine, newLength,
+        mtime, truncateBlock);
+    return new TruncateResult(onBlockBoundary, fsd.getAuditFileInfo(iip));
+  }
+
+  /**
+   * Unprotected truncate implementation. Unlike
+   * {@link FSDirTruncateOp#truncate}, this will not schedule block recovery.
+   *
+   * @param fsn namespace
+   * @param src path name
+   * @param clientName client name
+   * @param clientMachine client machine info
+   * @param newLength the target file size
+   * @param mtime modified time
+   * @param truncateBlock truncate block
+   * @throws IOException
+   */
+  static void unprotectedTruncate(final FSNamesystem fsn, final String src,
+      final String clientName, final String clientMachine,
+      final long newLength, final long mtime, final Block truncateBlock)
+      throws UnresolvedLinkException, QuotaExceededException,
+      SnapshotAccessControlException, IOException {
+    assert fsn.hasWriteLock();
+
+    FSDirectory fsd = fsn.getFSDirectory();
+    INodesInPath iip = fsd.getINodesInPath(src, true);
+    INodeFile file = iip.getLastINode().asFile();
+    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+    boolean onBlockBoundary = unprotectedTruncate(fsn, iip, newLength,
+        collectedBlocks, mtime, null);
+
+    if (!onBlockBoundary) {
+      BlockInfo oldBlock = file.getLastBlock();
+      Block tBlk = prepareFileForTruncate(fsn, iip, clientName, clientMachine,
+          file.computeFileSize() - newLength, truncateBlock);
+      assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
+          tBlk.getNumBytes() == truncateBlock.getNumBytes() :
+          "Should be the same block.";
+      if (oldBlock.getBlockId() != tBlk.getBlockId()
+          && !file.isBlockInLatestSnapshot(oldBlock)) {
+        fsn.getBlockManager().removeBlockFromMap(oldBlock);
+      }
+    }
+    assert onBlockBoundary == (truncateBlock == null) :
+      "truncateBlock is null iff on block boundary: " + truncateBlock;
+    fsn.removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+  }
+
+  /**
+   * Convert current INode to UnderConstruction. Recreate lease. Create new
+   * block for the truncated copy. Schedule truncation of the replicas.
+   *
+   * @param fsn namespace
+   * @param iip inodes in the path containing the file
+   * @param leaseHolder lease holder
+   * @param clientMachine client machine info
+   * @param lastBlockDelta last block delta size
+   * @param newBlock new block
+   * @return the returned block will be written to editLog and passed back
+   *         into this method upon loading.
+   * @throws IOException
+   */
+  @VisibleForTesting
+  static Block prepareFileForTruncate(FSNamesystem fsn, INodesInPath iip,
+      String leaseHolder, String clientMachine, long lastBlockDelta,
+      Block newBlock) throws IOException {
+    assert fsn.hasWriteLock();
+
+    INodeFile file = iip.getLastINode().asFile();
+    file.recordModification(iip.getLatestSnapshotId());
+    file.toUnderConstruction(leaseHolder, clientMachine);
+    assert file.isUnderConstruction() : "inode should be under construction.";
+    fsn.getLeaseManager().addLease(
+        file.getFileUnderConstructionFeature().getClientName(), file.getId());
+    boolean shouldRecoverNow = (newBlock == null);
+    BlockInfo oldBlock = file.getLastBlock();
+    boolean shouldCopyOnTruncate = shouldCopyOnTruncate(fsn, file, oldBlock);
+    if (newBlock == null) {
+      newBlock = (shouldCopyOnTruncate) ? fsn.createNewBlock() : new Block(
+          oldBlock.getBlockId(), oldBlock.getNumBytes(),
+          fsn.nextGenerationStamp(fsn.getBlockIdManager().isLegacyBlock(
+              oldBlock)));
+    }
+
+    BlockInfoUnderConstruction truncatedBlockUC;
+    if (shouldCopyOnTruncate) {
+      // Add new truncateBlock into blocksMap and
+      // use oldBlock as a source for copy-on-truncate recovery
+      truncatedBlockUC = new BlockInfoUnderConstructionContiguous(newBlock,
+          file.getPreferredBlockReplication());
+      truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
+      truncatedBlockUC.setTruncateBlock(oldBlock);
+      file.setLastBlock(truncatedBlockUC,
+          fsn.getBlockManager().getStorages(oldBlock));
+      fsn.getBlockManager().addBlockCollection(truncatedBlockUC, file);
+
+      NameNode.stateChangeLog.debug(
+          "BLOCK* prepareFileForTruncate: Scheduling copy-on-truncate to new"
+              + " size {}  new block {} old block {}",
+          truncatedBlockUC.getNumBytes(), newBlock,
+          truncatedBlockUC.getTruncateBlock());
+    } else {
+      // Use new generation stamp for in-place truncate recovery
+      fsn.getBlockManager().convertLastBlockToUnderConstruction(file,
+          lastBlockDelta);
+      oldBlock = file.getLastBlock();
+      assert !oldBlock.isComplete() : "oldBlock should be under construction";
+      truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
+      truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
+      truncatedBlockUC.getTruncateBlock().setNumBytes(
+          oldBlock.getNumBytes() - lastBlockDelta);
+      truncatedBlockUC.getTruncateBlock().setGenerationStamp(
+          newBlock.getGenerationStamp());
+
+      NameNode.stateChangeLog.debug(
+          "BLOCK* prepareFileForTruncate: {} Scheduling in-place block "
+              + "truncate to new size {}", truncatedBlockUC.getTruncateBlock()
+              .getNumBytes(), truncatedBlockUC);
+    }
+    if (shouldRecoverNow) {
+      truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
+    }
+
+    return newBlock;
+  }
+
+  /**
+   * Truncate has the following properties:
+   * 1.) Any block deletions occur now.
+   * 2.) INode length is truncated now - new clients can only read up to
+   *     the truncated length.
+   * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
+   * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
+   * 5.) File is considered UNDER_RECOVERY until truncation recovery
+   *     completes.
+   * 6.) Soft and hard Lease expiration require truncation recovery to
+   *     complete.
+   *
+   * @return true if on the block boundary or false if recovery is need
+   */
+  private static boolean unprotectedTruncate(FSNamesystem fsn,
+      INodesInPath iip, long newLength, BlocksMapUpdateInfo collectedBlocks,
+      long mtime, QuotaCounts delta) throws IOException {
+    assert fsn.hasWriteLock();
+
+    INodeFile file = iip.getLastINode().asFile();
+    int latestSnapshot = iip.getLatestSnapshotId();
+    file.recordModification(latestSnapshot, true);
+
+    verifyQuotaForTruncate(fsn, iip, file, newLength, delta);
+
+    long remainingLength =
+        file.collectBlocksBeyondMax(newLength, collectedBlocks);
+    file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
+    file.setModificationTime(mtime);
+    // return whether on a block boundary
+    return (remainingLength - newLength) == 0;
+  }
+
+  private static void verifyQuotaForTruncate(FSNamesystem fsn,
+      INodesInPath iip, INodeFile file, long newLength, QuotaCounts delta)
+      throws QuotaExceededException {
+    FSDirectory fsd = fsn.getFSDirectory();
+    if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
+      // Do not check quota if edit log is still being processed
+      return;
+    }
+    final BlockStoragePolicy policy = fsd.getBlockStoragePolicySuite()
+        .getPolicy(file.getStoragePolicyID());
+    file.computeQuotaDeltaForTruncate(newLength, policy, delta);
+    fsd.readLock();
+    try {
+      FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null);
+    } finally {
+      fsd.readUnlock();
+    }
+  }
+
+  /**
+   * Defines if a replica needs to be copied on truncate or
+   * can be truncated in place.
+   */
+  private static boolean shouldCopyOnTruncate(FSNamesystem fsn, INodeFile file,
+      BlockInfo blk) {
+    if (!fsn.isUpgradeFinalized()) {
+      return true;
+    }
+    if (fsn.isRollingUpgrade()) {
+      return true;
+    }
+    return file.isBlockInLatestSnapshot(blk);
+  }
+
+  /**
+   * Result of truncate operation.
+   */
+  static class TruncateResult {
+    private final boolean result;
+    private final HdfsFileStatus stat;
+
+    public TruncateResult(boolean result, HdfsFileStatus stat) {
+      this.result = result;
+      this.stat = stat;
+    }
+
+    /**
+     * @return true if client does not need to wait for block recovery,
+     *          false if client needs to wait for block recovery.
+     */
+    boolean getResult() {
+      return result;
+    }
+
+    /**
+     * @return file information.
+     */
+    HdfsFileStatus getFileStatus() {
+      return stat;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c807fba..ccee1ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
@@ -49,11 +48,9 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 import org.apache.hadoop.hdfs.util.ByteArray;
 import org.apache.hadoop.hdfs.util.EnumCounters;
 import org.apache.hadoop.security.AccessControlException;
@@ -908,98 +905,6 @@ public class FSDirectory implements Closeable {
   }
 
   /**
-   * FSEditLogLoader implementation.
-   * Unlike FSNamesystem.truncate, this will not schedule block recovery.
-   */
-  void unprotectedTruncate(String src, String clientName, String clientMachine,
-                           long newLength, long mtime, Block truncateBlock)
-      throws UnresolvedLinkException, QuotaExceededException,
-      SnapshotAccessControlException, IOException {
-    INodesInPath iip = getINodesInPath(src, true);
-    INodeFile file = iip.getLastINode().asFile();
-    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
-    boolean onBlockBoundary =
-        unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null);
-
-    if(! onBlockBoundary) {
-      BlockInfo oldBlock = file.getLastBlock();
-      Block tBlk =
-      getFSNamesystem().prepareFileForTruncate(iip,
-          clientName, clientMachine, file.computeFileSize() - newLength,
-          truncateBlock);
-      assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
-          tBlk.getNumBytes() == truncateBlock.getNumBytes() :
-          "Should be the same block.";
-      if(oldBlock.getBlockId() != tBlk.getBlockId() &&
-         !file.isBlockInLatestSnapshot(oldBlock)) {
-        getBlockManager().removeBlockFromMap(oldBlock);
-      }
-    }
-    assert onBlockBoundary == (truncateBlock == null) :
-      "truncateBlock is null iff on block boundary: " + truncateBlock;
-    getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
-  }
-
-  boolean truncate(INodesInPath iip, long newLength,
-                   BlocksMapUpdateInfo collectedBlocks,
-                   long mtime, QuotaCounts delta)
-      throws IOException {
-    writeLock();
-    try {
-      return unprotectedTruncate(iip, newLength, collectedBlocks, mtime, delta);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  /**
-   * Truncate has the following properties:
-   * 1.) Any block deletions occur now.
-   * 2.) INode length is truncated now – new clients can only read up to
-   * the truncated length.
-   * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
-   * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
-   * 5.) File is considered UNDER_RECOVERY until truncation recovery completes.
-   * 6.) Soft and hard Lease expiration require truncation recovery to complete.
-   *
-   * @return true if on the block boundary or false if recovery is need
-   */
-  boolean unprotectedTruncate(INodesInPath iip, long newLength,
-                              BlocksMapUpdateInfo collectedBlocks,
-                              long mtime, QuotaCounts delta) throws IOException {
-    assert hasWriteLock();
-    INodeFile file = iip.getLastINode().asFile();
-    int latestSnapshot = iip.getLatestSnapshotId();
-    file.recordModification(latestSnapshot, true);
-
-    verifyQuotaForTruncate(iip, file, newLength, delta);
-
-    long remainingLength =
-        file.collectBlocksBeyondMax(newLength, collectedBlocks);
-    file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
-    file.setModificationTime(mtime);
-    // return whether on a block boundary
-    return (remainingLength - newLength) == 0;
-  }
-
-  private void verifyQuotaForTruncate(INodesInPath iip, INodeFile file,
-      long newLength, QuotaCounts delta) throws QuotaExceededException {
-    if (!getFSNamesystem().isImageLoaded() || shouldSkipQuotaChecks()) {
-      // Do not check quota if edit log is still being processed
-      return;
-    }
-    final BlockStoragePolicy policy = getBlockStoragePolicySuite()
-        .getPolicy(file.getStoragePolicyID());
-    file.computeQuotaDeltaForTruncate(newLength, policy, delta);
-    readLock();
-    try {
-      verifyQuota(iip, iip.length() - 1, delta, null);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /**
    * This method is always called with writeLock of FSDirectory held.
    */
   public final void addToInodeMap(INode inode) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index df01edd..63ef985 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -901,9 +901,9 @@ public class FSEditLogLoader {
     }
     case OP_TRUNCATE: {
       TruncateOp truncateOp = (TruncateOp) op;
-      fsDir.unprotectedTruncate(truncateOp.src, truncateOp.clientName,
-          truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp,
-          truncateOp.truncateBlock);
+      FSDirTruncateOp.unprotectedTruncate(fsNamesys, truncateOp.src,
+          truncateOp.clientName, truncateOp.clientMachine,
+          truncateOp.newLength, truncateOp.timestamp, truncateOp.truncateBlock);
       break;
     }
     case OP_SET_STORAGE_POLICY: {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e95007b..7c6d6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -201,7 +201,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -1831,218 +1830,44 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    * block recovery to truncate the last block of the file.
    *
    * @return true if client does not need to wait for block recovery,
-   * false if client needs to wait for block recovery.
+   *         false if client needs to wait for block recovery.
    */
-  boolean truncate(String src, long newLength,
-                   String clientName, String clientMachine,
-                   long mtime)
-      throws IOException, UnresolvedLinkException {
+  boolean truncate(String src, long newLength, String clientName,
+      String clientMachine, long mtime) throws IOException,
+      UnresolvedLinkException {
+
     requireEffectiveLayoutVersionForFeature(Feature.TRUNCATE);
-    boolean ret;
+    final FSDirTruncateOp.TruncateResult r;
     try {
-      ret = truncateInt(src, newLength, clientName, clientMachine, mtime);
+      NameNode.stateChangeLog.debug(
+          "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
+      if (newLength < 0) {
+        throw new HadoopIllegalArgumentException(
+            "Cannot truncate to a negative file size: " + newLength + ".");
+      }
+      final FSPermissionChecker pc = getPermissionChecker();
+      checkOperation(OperationCategory.WRITE);
+      writeLock();
+      BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
+      try {
+        checkOperation(OperationCategory.WRITE);
+        checkNameNodeSafeMode("Cannot truncate for " + src);
+        r = FSDirTruncateOp.truncate(this, src, newLength, clientName,
+            clientMachine, mtime, toRemoveBlocks, pc);
+      } finally {
+        writeUnlock();
+      }
+      getEditLog().logSync();
+      if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
+        removeBlocks(toRemoveBlocks);
+        toRemoveBlocks.clear();
+      }
+      logAuditEvent(true, "truncate", src, null, r.getFileStatus());
     } catch (AccessControlException e) {
       logAuditEvent(false, "truncate", src);
       throw e;
     }
-    return ret;
-  }
-
-  boolean truncateInt(String srcArg, long newLength,
-                      String clientName, String clientMachine,
-                      long mtime)
-      throws IOException, UnresolvedLinkException {
-    String src = srcArg;
-    NameNode.stateChangeLog.debug(
-        "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
-    if (newLength < 0) {
-      throw new HadoopIllegalArgumentException(
-          "Cannot truncate to a negative file size: " + newLength + ".");
-    }
-    HdfsFileStatus stat = null;
-    FSPermissionChecker pc = getPermissionChecker();
-    checkOperation(OperationCategory.WRITE);
-    boolean res;
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
-    writeLock();
-    BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
-    try {
-      checkOperation(OperationCategory.WRITE);
-      checkNameNodeSafeMode("Cannot truncate for " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      res = truncateInternal(src, newLength, clientName,
-          clientMachine, mtime, pc, toRemoveBlocks);
-      stat = dir.getAuditFileInfo(dir.getINodesInPath4Write(src, false));
-    } finally {
-      writeUnlock();
-    }
-    getEditLog().logSync();
-    if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
-      removeBlocks(toRemoveBlocks);
-      toRemoveBlocks.clear();
-    }
-    logAuditEvent(true, "truncate", src, null, stat);
-    return res;
-  }
-
-  /**
-   * Truncate a file to a given size
-   * Update the count at each ancestor directory with quota
-   */
-  boolean truncateInternal(String src, long newLength,
-                           String clientName, String clientMachine,
-                           long mtime, FSPermissionChecker pc,
-                           BlocksMapUpdateInfo toRemoveBlocks)
-      throws IOException, UnresolvedLinkException {
-    assert hasWriteLock();
-    INodesInPath iip = dir.getINodesInPath4Write(src, true);
-    if (isPermissionEnabled) {
-      dir.checkPathAccess(pc, iip, FsAction.WRITE);
-    }
-    INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
-    final BlockStoragePolicy lpPolicy =
-        blockManager.getStoragePolicy("LAZY_PERSIST");
-
-    if (lpPolicy != null &&
-        lpPolicy.getId() == file.getStoragePolicyID()) {
-      throw new UnsupportedOperationException(
-          "Cannot truncate lazy persist file " + src);
-    }
-
-    // Check if the file is already being truncated with the same length
-    final BlockInfo last = file.getLastBlock();
-    if (last != null && last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
-      final Block truncateBlock
-          = ((BlockInfoUnderConstruction)last).getTruncateBlock();
-      if (truncateBlock != null) {
-        final long truncateLength = file.computeFileSize(false, false)
-            + truncateBlock.getNumBytes();
-        if (newLength == truncateLength) {
-          return false;
-        }
-      }
-    }
-
-    // Opening an existing file for truncate. May need lease recovery.
-    recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE,
-        iip, src, clientName, clientMachine, false);
-    // Truncate length check.
-    long oldLength = file.computeFileSize();
-    if(oldLength == newLength) {
-      return true;
-    }
-    if(oldLength < newLength) {
-      throw new HadoopIllegalArgumentException(
-          "Cannot truncate to a larger file size. Current size: " + oldLength +
-              ", truncate size: " + newLength + ".");
-    }
-    // Perform INodeFile truncation.
-    final QuotaCounts delta = new QuotaCounts.Builder().build();
-    boolean onBlockBoundary = dir.truncate(iip, newLength, toRemoveBlocks,
-        mtime, delta);
-    Block truncateBlock = null;
-    if(!onBlockBoundary) {
-      // Open file for write, but don't log into edits
-      long lastBlockDelta = file.computeFileSize() - newLength;
-      assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
-      truncateBlock = prepareFileForTruncate(iip, clientName, clientMachine,
-          lastBlockDelta, null);
-    }
-
-    // update the quota: use the preferred block size for UC block
-    dir.writeLock();
-    try {
-      dir.updateCountNoQuotaCheck(iip, iip.length() - 1, delta);
-    } finally {
-      dir.writeUnlock();
-    }
-
-    getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime,
-        truncateBlock);
-    return onBlockBoundary;
-  }
-
-  /**
-   * Convert current INode to UnderConstruction.
-   * Recreate lease.
-   * Create new block for the truncated copy.
-   * Schedule truncation of the replicas.
-   *
-   * @return the returned block will be written to editLog and passed back into
-   * this method upon loading.
-   */
-  Block prepareFileForTruncate(INodesInPath iip,
-                               String leaseHolder,
-                               String clientMachine,
-                               long lastBlockDelta,
-                               Block newBlock)
-      throws IOException {
-    INodeFile file = iip.getLastINode().asFile();
-    file.recordModification(iip.getLatestSnapshotId());
-    file.toUnderConstruction(leaseHolder, clientMachine);
-    assert file.isUnderConstruction() : "inode should be under construction.";
-    leaseManager.addLease(
-        file.getFileUnderConstructionFeature().getClientName(), file.getId());
-    boolean shouldRecoverNow = (newBlock == null);
-    BlockInfo oldBlock = file.getLastBlock();
-    boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
-    if(newBlock == null) {
-      newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
-          new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
-              nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock)));
-    }
-
-    BlockInfoUnderConstruction truncatedBlockUC;
-    if(shouldCopyOnTruncate) {
-      // Add new truncateBlock into blocksMap and
-      // use oldBlock as a source for copy-on-truncate recovery
-      truncatedBlockUC = new BlockInfoUnderConstructionContiguous(newBlock,
-          file.getPreferredBlockReplication());
-      truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
-      truncatedBlockUC.setTruncateBlock(oldBlock);
-      file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock));
-      getBlockManager().addBlockCollection(truncatedBlockUC, file);
-
-      NameNode.stateChangeLog.debug(
-          "BLOCK* prepareFileForTruncate: Scheduling copy-on-truncate to new" +
-          " size {}  new block {} old block {}", truncatedBlockUC.getNumBytes(),
-          newBlock, truncatedBlockUC.getTruncateBlock());
-    } else {
-      // Use new generation stamp for in-place truncate recovery
-      blockManager.convertLastBlockToUnderConstruction(file, lastBlockDelta);
-      oldBlock = file.getLastBlock();
-      assert !oldBlock.isComplete() : "oldBlock should be under construction";
-      truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
-      truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
-      truncatedBlockUC.getTruncateBlock().setNumBytes(
-          oldBlock.getNumBytes() - lastBlockDelta);
-      truncatedBlockUC.getTruncateBlock().setGenerationStamp(
-          newBlock.getGenerationStamp());
-
-      NameNode.stateChangeLog.debug(
-          "BLOCK* prepareFileForTruncate: {} Scheduling in-place block " +
-          "truncate to new size {}",
-          truncatedBlockUC.getTruncateBlock().getNumBytes(), truncatedBlockUC);
-    }
-    if (shouldRecoverNow) {
-      truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
-    }
-
-    return newBlock;
-  }
-
-  /**
-   * Defines if a replica needs to be copied on truncate or
-   * can be truncated in place.
-   */
-  boolean shouldCopyOnTruncate(INodeFile file, BlockInfo blk) {
-    if(!isUpgradeFinalized()) {
-      return true;
-    }
-    if (isRollingUpgrade()) {
-      return true;
-    }
-    return file.isBlockInLatestSnapshot(blk);
+    return r.getResult();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e39b962/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index df920e0..e0f9ad2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -1008,8 +1008,8 @@ public class TestFileTruncate {
     fsn.writeLock();
     try {
       Block oldBlock = file.getLastBlock();
-      Block truncateBlock =
-          fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+      Block truncateBlock = FSDirTruncateOp.prepareFileForTruncate(fsn, iip,
+          client, clientMachine, 1, null);
       // In-place truncate uses old block id with new genStamp.
       assertThat(truncateBlock.getBlockId(),
           is(equalTo(oldBlock.getBlockId())));
@@ -1041,8 +1041,8 @@ public class TestFileTruncate {
     fsn.writeLock();
     try {
       Block oldBlock = file.getLastBlock();
-      Block truncateBlock =
-          fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+      Block truncateBlock = FSDirTruncateOp.prepareFileForTruncate(fsn, iip,
+          client, clientMachine, 1, null);
       // Copy-on-write truncate makes new block with new id and genStamp
       assertThat(truncateBlock.getBlockId(),
           is(not(equalTo(oldBlock.getBlockId()))));


[18/48] hadoop git commit: HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)

Posted by zj...@apache.org.
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b402281
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b402281
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b402281

Branch: refs/heads/YARN-2928
Commit: 5b4022815ed8ce2b2777388682474fa775849881
Parents: 8ed2be0
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 15:54:38 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../hadoop-hdfs/src/CMakeLists.txt              | 129 ++++++-------------
 .../src/contrib/libwebhdfs/CMakeLists.txt       |   8 +-
 .../libwebhdfs/resources/FindJansson.cmake      |   9 +-
 .../src/main/native/fuse-dfs/CMakeLists.txt     |  91 +++++++------
 5 files changed, 100 insertions(+), 140 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b065f98..65f286d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8493. Consolidate truncate() related implementation in a single class.
     (Rakesh R via wheat9)
 
+    HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
+    via Colin P. McCabe)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 563727b..b4a3b40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -18,106 +18,59 @@
 
 cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
 
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
-    add_library(${LIBNAME} SHARED ${ARGN})
-    add_library(${LIBNAME}_static STATIC ${ARGN})
-    # Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so
-    # (dynamic linking).  On Windows, we cannot use the same base name for both
-    # static and dynamic, because Windows does not use distinct file extensions
-    # for a statically linked library vs. a DLL import library.  Both use the
-    # .lib extension.  On Windows, we'll build the static library as
-    # hdfs_static.lib.
-    if (NOT WIN32)
-        set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
-    endif (NOT WIN32)
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
-    target_link_libraries(${LIBNAME} ${ARGN})
-    target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
-    output_directory(${TGT} "${DIR}")
-    output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
+#
+# Main configuration
+#
 
-# Flatten a list into a string.
-function(FLATTEN_LIST INPUT SEPARATOR OUTPUT)
-  string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
-  set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
-endfunction()
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+    message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
 
 # Check to see if our compiler and linker support the __thread attribute.
-# On Linux and some other operating systems, this is a more efficient 
-# alternative to POSIX thread local storage. 
-INCLUDE(CheckCSourceCompiles)
-CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+# On Linux and some other operating systems, this is a more efficient
+# alternative to POSIX thread local storage.
+include(CheckCSourceCompiles)
+check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
 
 # Check to see if we have Intel SSE intrinsics.
-CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
+check_c_source_compileS("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
 
 # Check if we need to link dl library to get dlopen.
 # dlopen on Linux is in separate library but on FreeBSD its in libc
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
 
-find_package(JNI REQUIRED)
-if (NOT GENERATED_JAVAH)
-    # Must identify where the generated headers have been placed
-    MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-
-if (WIN32)
+if(WIN32)
+    # Set the optimizer level.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2")
-
     # Set warning level 4.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
-
     # Skip "unreferenced formal parameter".
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100")
-
     # Skip "conditional expression is constant".
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127")
-
     # Skip deprecated POSIX function warnings.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE")
-
     # Skip CRT non-secure function warnings.  If we can convert usage of
     # strerror, getenv and ctime to their secure CRT equivalents, then we can
     # re-enable the CRT non-secure function warnings.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS")
-
     # Omit unneeded headers.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
-
     set(OS_DIR main/native/libhdfs/os/windows)
     set(OUT_DIR target/bin)
-else (WIN32)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
+else()
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
     set(OS_DIR main/native/libhdfs/os/posix)
-    set(OS_LINK_LIBRARIES pthread)
     set(OUT_DIR target/usr/local/lib)
-endif (WIN32)
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
 
 add_definitions(-DLIBHDFS_DLL_EXPORT)
 
@@ -132,9 +85,9 @@ include_directories(
 )
 
 set(_FUSE_DFS_VERSION 0.1.0)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
 
-add_dual_library(hdfs
+hadoop_add_dual_library(hdfs
     main/native/libhdfs/exception.c
     main/native/libhdfs/jni_helper.c
     main/native/libhdfs/hdfs.c
@@ -142,17 +95,17 @@ add_dual_library(hdfs
     ${OS_DIR}/mutexes.c
     ${OS_DIR}/thread_local_storage.c
 )
-if (NEED_LINK_DL)
+if(NEED_LINK_DL)
    set(LIB_DL dl)
-endif(NEED_LINK_DL)
+endif()
 
-target_link_dual_libraries(hdfs
+hadoop_target_link_dual_libraries(hdfs
     ${JAVA_JVM_LIBRARY}
     ${LIB_DL}
     ${OS_LINK_LIBRARIES}
 )
 
-dual_output_directory(hdfs ${OUT_DIR})
+hadoop_dual_output_directory(hdfs ${OUT_DIR})
 set(LIBHDFS_VERSION "0.0.0")
 set_target_properties(hdfs PROPERTIES
     SOVERSION ${LIBHDFS_VERSION})
@@ -232,25 +185,21 @@ target_link_libraries(test_htable
 
 # Skip vecsum on Windows.  This could be made to work in the future by
 # introducing an abstraction layer over the sys/mman.h functions.
-if (NOT WIN32)
-    add_executable(test_libhdfs_vecsum
-        main/native/libhdfs/test/vecsum.c
-    )
-    if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(NOT WIN32)
+    add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c)
+    if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
         target_link_libraries(test_libhdfs_vecsum
             hdfs
-            pthread
-        )
-    else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+            pthread)
+    else()
         target_link_libraries(test_libhdfs_vecsum
             hdfs
             pthread
-            rt
-        )
-    endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-endif(NOT WIN32)
+            rt)
+    endif()
+endif()
 
-IF(REQUIRE_LIBWEBHDFS)
+if(REQUIRE_LIBWEBHDFS)
     add_subdirectory(contrib/libwebhdfs)
-ENDIF(REQUIRE_LIBWEBHDFS)
+endif()
 add_subdirectory(main/native/fuse-dfs)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
index 44f0c42..dc74feb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
@@ -19,12 +19,12 @@
 find_package(CURL REQUIRED)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
+    "${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
 
 find_package(Jansson REQUIRED)
 include_directories(${JANSSON_INCLUDE_DIR})
 
-add_dual_library(webhdfs
+hadoop_add_dual_library(webhdfs
     src/hdfs_web.c
     src/hdfs_http_client.c
     src/hdfs_http_query.c
@@ -35,13 +35,13 @@ add_dual_library(webhdfs
     ../../${OS_DIR}/thread_local_storage.c
     ../../main/native/libhdfs/common/htable.c
 )
-target_link_dual_libraries(webhdfs
+hadoop_target_link_dual_libraries(webhdfs
     ${JAVA_JVM_LIBRARY}
     ${CURL_LIBRARY}
     ${JANSSON_LIBRARY}
     pthread
 )
-dual_output_directory(webhdfs target)
+hadoop_dual_output_directory(webhdfs target)
 set(LIBWEBHDFS_VERSION "0.0.0")
 set_target_properties(webhdfs PROPERTIES
     SOVERSION ${LIBWEBHDFS_VERSION})

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
index cb8c8db..b8c67ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,8 @@
 #  JANSSON_DEFINITIONS - Compiler switches required for using Jansson
 
 find_path(JANSSON_INCLUDE_DIR jansson.h
-          /usr/incluce
+          /usr/include
+          /usr/include/jansson
           /usr/local/include )
 
 find_library(JANSSON_LIBRARY NAMES jansson
@@ -39,4 +40,4 @@ include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(Jansson  DEFAULT_MSG
                                   JANSSON_LIBRARY JANSSON_INCLUDE_DIR)
 
-mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
\ No newline at end of file
+mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b402281/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
index 8a8ea5b..7a5e5a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
@@ -18,57 +18,63 @@
 
 set(CMAKE_SKIP_RPATH TRUE)
 
+# Flatten a list into a string.
+function(flatten_list INPUT SEPARATOR OUTPUT)
+  string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
+  set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
+endfunction()
+
 # Find Linux FUSE
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     find_package(PkgConfig REQUIRED)
     pkg_check_modules(FUSE fuse)
-    IF(FUSE_FOUND)
-        FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
-        FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
+    if(FUSE_FOUND)
+        flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
+        flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}")
         set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}")
-        MESSAGE(STATUS "Building Linux FUSE client.")
+        message(STATUS "Building Linux FUSE client.")
         include_directories(${FUSE_INCLUDE_DIRS})
-    ELSE(FUSE_FOUND)
-        MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files.  Will not build FUSE client.")
-    ENDIF(FUSE_FOUND)
-ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
-    MESSAGE(STATUS "Non-Linux system detected.  Will not build FUSE client.")
-ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+    else()
+        message(STATUS "Failed to find Linux FUSE libraries or include files.  Will not build FUSE client.")
+    endif(FUSE_FOUND)
+else()
+    message(STATUS "Non-Linux system detected.  Will not build FUSE client.")
+endif()
 
-IF(FUSE_FOUND)
+if(FUSE_FOUND)
     add_library(posix_util
         ../util/posix_util.c
     )
 
     add_executable(fuse_dfs
         fuse_dfs.c
-        fuse_options.c 
-        fuse_connect.c 
-        fuse_impls_access.c 
-        fuse_impls_chmod.c  
-        fuse_impls_chown.c  
-        fuse_impls_create.c  
-        fuse_impls_flush.c 
-        fuse_impls_getattr.c  
-        fuse_impls_mkdir.c  
-        fuse_impls_mknod.c  
-        fuse_impls_open.c 
-        fuse_impls_read.c 
-        fuse_impls_readdir.c 
-        fuse_impls_release.c 
-        fuse_impls_rename.c 
-        fuse_impls_rmdir.c 
-        fuse_impls_statfs.c 
-        fuse_impls_symlink.c 
-        fuse_impls_truncate.c 
-        fuse_impls_unlink.c 
-        fuse_impls_utimens.c  
+        fuse_options.c
+        fuse_connect.c
+        fuse_impls_access.c
+        fuse_impls_chmod.c
+        fuse_impls_chown.c
+        fuse_impls_create.c
+        fuse_impls_flush.c
+        fuse_impls_getattr.c
+        fuse_impls_mkdir.c
+        fuse_impls_mknod.c
+        fuse_impls_open.c
+        fuse_impls_read.c
+        fuse_impls_readdir.c
+        fuse_impls_release.c
+        fuse_impls_rename.c
+        fuse_impls_rmdir.c
+        fuse_impls_statfs.c
+        fuse_impls_symlink.c
+        fuse_impls_truncate.c
+        fuse_impls_unlink.c
+        fuse_impls_utimens.c
         fuse_impls_write.c
-        fuse_init.c 
-        fuse_stat_struct.c 
-        fuse_trash.c 
-        fuse_users.c 
+        fuse_init.c
+        fuse_stat_struct.c
+        fuse_trash.c
+        fuse_users.c
     )
     target_link_libraries(fuse_dfs
         ${FUSE_LIBRARIES}
@@ -88,8 +94,9 @@ IF(FUSE_FOUND)
         posix_util
         pthread
     )
-ELSE(FUSE_FOUND)
-    IF(REQUIRE_FUSE)
-        MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.")
-    ENDIF(REQUIRE_FUSE)
-ENDIF(FUSE_FOUND)
+    set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning
+else()
+    if(REQUIRE_FUSE)
+        message(FATAL_ERROR "Required component fuse_dfs could not be built.")
+    endif()
+endif()


[31/48] hadoop git commit: MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be logged in warn/info level. Contributed by Chang Li

Posted by zj...@apache.org.
MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be logged in warn/info level. Contributed by Chang Li


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef7b82be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef7b82be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef7b82be

Branch: refs/heads/YARN-2928
Commit: ef7b82be1281f1c64bab9bfc6d90ac334fe62868
Parents: d57f65e
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 17:50:02 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt                              | 3 +++
 .../java/org/apache/hadoop/mapred/LocalContainerLauncher.java     | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef7b82be/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 49a873d..dd739ca 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
     (rchiang via rkanter)
 
+    MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be
+    logged in warn/info level (Chang Li via jlowe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef7b82be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 9d8b4a5..1a0d5fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -233,7 +233,7 @@ public class LocalContainerLauncher extends AbstractService implements
         try {
           event = eventQueue.take();
         } catch (InterruptedException e) {  // mostly via T_KILL? JOB_KILL?
-          LOG.error("Returning, interrupted : " + e);
+          LOG.warn("Returning, interrupted : " + e);
           break;
         }
 


[17/48] hadoop git commit: HADOOP-12158. Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed.

Posted by zj...@apache.org.
HADOOP-12158. Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ed2be02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ed2be02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ed2be02

Branch: refs/heads/YARN-2928
Commit: 8ed2be02f9817bf3de136cb53ed23c016cfd067d
Parents: bcac9c9
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Jun 30 15:04:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt             | 5 +++++
 .../TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java      | 9 +++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed2be02/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e332ea8..2aab8b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -666,6 +666,11 @@ Release 2.8.0 - UNRELEASED
 
     HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
     occurs (Jason Lowe via jeagles)
+
+    HADOOP-12158. Improve error message in
+    TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not
+    installed. (wang)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed2be02/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index 4f90a0c..684ec09 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.crypto;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.junit.Assert;
 import org.junit.BeforeClass;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
 public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec 
     extends TestCryptoStreams {
   
@@ -32,7 +34,10 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
         CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
         OpensslAesCtrCryptoCodec.class.getName());
     codec = CryptoCodec.getInstance(conf);
-    Assert.assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(), 
+    assertNotNull("Unable to instantiate codec " +
+        OpensslAesCtrCryptoCodec.class.getName() + ", is the required "
+        + "version of OpenSSL installed?", codec);
+    assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
         codec.getClass().getCanonicalName());
   }
 }


[05/48] hadoop git commit: HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework (Alan Burlison via Colin P. McCabe)

Posted by zj...@apache.org.
HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework (Alan Burlison via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96d07d2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96d07d2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96d07d2d

Branch: refs/heads/YARN-2928
Commit: 96d07d2dbae3f1a30cc178ee237402369d034350
Parents: e9d1005
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 29 12:01:17 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 hadoop-tools/hadoop-pipes/src/CMakeLists.txt    | 49 +++++++++-----------
 2 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d07d2d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0a964a3..e8e85a0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash.
     (wang)
 
+    HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework
+    (alanburlison via cmccabe)
+
     HADOOP-12036. Consolidate all of the cmake extensions in one directory
     (alanburlison via cmccabe)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d07d2d/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
index 170af49..3b0b28c 100644
--- a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
+++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
@@ -6,7 +6,7 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
- 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ -17,25 +17,11 @@
 #
 
 cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-find_package(OpenSSL REQUIRED)
-
-set(CMAKE_BUILD_TYPE, Release)
 
-set(PIPES_FLAGS "-g -Wall -O2 -D_REENTRANT -D_GNU_SOURCE")
-set(PIPES_FLAGS "${PIPES_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PIPES_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PIPES_FLAGS}")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
 
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-function(output_directory TGT DIR)
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-   SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
+find_package(OpenSSL REQUIRED)
 
 include_directories(
     main/native/utils/api
@@ -47,19 +33,19 @@ include_directories(
 # Example programs
 add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
 target_link_libraries(wordcount-simple hadooppipes hadooputils)
-output_directory(wordcount-simple examples)
+hadoop_output_directory(wordcount-simple examples)
 
 add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
 target_link_libraries(wordcount-part hadooppipes hadooputils)
-output_directory(wordcount-part examples)
+hadoop_output_directory(wordcount-part examples)
 
 add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
 target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
-output_directory(wordcount-nopipe examples)
+hadoop_output_directory(wordcount-nopipe examples)
 
 add_executable(pipes-sort main/native/examples/impl/sort.cc)
 target_link_libraries(pipes-sort hadooppipes hadooputils)
-output_directory(pipes-sort examples)
+hadoop_output_directory(pipes-sort examples)
 
 add_library(hadooputils STATIC
     main/native/utils/impl/StringUtils.cc
@@ -70,15 +56,22 @@ add_library(hadooppipes STATIC
     main/native/pipes/impl/HadoopPipes.cc
 )
 
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
+
+if(NEED_LINK_DL)
+    set(LIB_DL "dl")
+endif()
 
-if (NEED_LINK_DL)
-    set(LIB_DL dl)
-endif (NEED_LINK_DL)
+if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+    exec_program("uname" ARGS "-r" OUTPUT_VARIABLE OS_VERSION)
+    if(OS_VERSION VERSION_LESS "5.12")
+        set(LIB_NET "socket" "nsl")
+    endif()
+endif()
 
 target_link_libraries(hadooppipes
     ${OPENSSL_LIBRARIES}
     ${LIB_DL}
-    pthread
+    ${LIB_NET}
 )


[24/48] hadoop git commit: YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison via Colin P. McCabe)

Posted by zj...@apache.org.
YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e713f2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e713f2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e713f2d

Branch: refs/heads/YARN-2928
Commit: 0e713f2dc654667a9e7616bd0d4b573de07681ce
Parents: 71a07c6
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:24:19 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +++
 .../src/CMakeLists.txt                          | 23 ++++++++++----------
 2 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e713f2d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 896f54e..5269bec 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -420,6 +420,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3834. Scrub debug logging of tokens during resource localization.
     (Chris Nauroth via xgong)
 
+    YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison
+    via Colin P. McCabe)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e713f2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index f691a9c..d4d6ae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -16,25 +16,24 @@
 
 cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
 
-include(../../../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
+# Note: can't use -D_FILE_OFFSET_BITS=64, see MAPREDUCE-4258
+string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 
 include(CheckFunctionExists)
-CHECK_FUNCTION_EXISTS(fcloseall HAVE_FCLOSEALL)
+check_function_exists(fcloseall HAVE_FCLOSEALL)
 
 function(output_directory TGT DIR)
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+    set_target_properties(${TGT} PROPERTIES
         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+    set_target_properties(${TGT} PROPERTIES
         ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+    set_target_properties(${TGT} PROPERTIES
         LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2 -D_GNU_SOURCE")
-# note: can't enable -D_LARGEFILE: see MAPREDUCE-4258
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT")
+endfunction()
 
 include_directories(
     ${CMAKE_CURRENT_SOURCE_DIR}
@@ -42,7 +41,7 @@ include_directories(
     main/native/container-executor
     main/native/container-executor/impl
 )
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
 
 add_library(container
     main/native/container-executor/impl/configuration.c


[27/48] hadoop git commit: MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via rkanter)

Posted by zj...@apache.org.
MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/36a33ebd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/36a33ebd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/36a33ebd

Branch: refs/heads/YARN-2928
Commit: 36a33ebdb5a456279861275a052f57b7884700ad
Parents: c906166
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 30 16:46:41 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 ++
 .../hadoop/mapreduce/JobResourceUploader.java   | 38 +-------------------
 2 files changed, 4 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a33ebd/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 5f4b3ef..d170b55 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host
     (zhihai xu via jlowe)
 
+    MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
+    (rchiang via rkanter)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/36a33ebd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 134de35..f3e4d2f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -194,7 +194,7 @@ class JobResourceUploader {
 
     FileSystem remoteFs = null;
     remoteFs = originalPath.getFileSystem(conf);
-    if (compareFs(remoteFs, jtFs)) {
+    if (FileUtil.compareFs(remoteFs, jtFs)) {
       return originalPath;
     }
     // this might have name collisions. copy will throw an exception
@@ -205,42 +205,6 @@ class JobResourceUploader {
     return newPath;
   }
 
-  /*
-   * see if two file systems are the same or not.
-   */
-  private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
-    URI srcUri = srcFs.getUri();
-    URI dstUri = destFs.getUri();
-    if (srcUri.getScheme() == null) {
-      return false;
-    }
-    if (!srcUri.getScheme().equals(dstUri.getScheme())) {
-      return false;
-    }
-    String srcHost = srcUri.getHost();
-    String dstHost = dstUri.getHost();
-    if ((srcHost != null) && (dstHost != null)) {
-      try {
-        srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
-        dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
-      } catch (UnknownHostException ue) {
-        return false;
-      }
-      if (!srcHost.equals(dstHost)) {
-        return false;
-      }
-    } else if (srcHost == null && dstHost != null) {
-      return false;
-    } else if (srcHost != null && dstHost == null) {
-      return false;
-    }
-    // check for ports
-    if (srcUri.getPort() != dstUri.getPort()) {
-      return false;
-    }
-    return true;
-  }
-
   private void copyJar(Path originalJarPath, Path submitJarFile,
       short replication) throws IOException {
     jtFs.copyFromLocalFile(originalJarPath, submitJarFile);


[10/48] hadoop git commit: HDFS-8659. Block scanner INFO message is spamming logs. Contributed by Yongjun Zhang.

Posted by zj...@apache.org.
HDFS-8659. Block scanner INFO message is spamming logs. Contributed by Yongjun Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94dca164
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94dca164
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94dca164

Branch: refs/heads/YARN-2928
Commit: 94dca164d625670ab37f9d810a676f5a1ab36c38
Parents: 296d150
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Jun 29 14:35:39 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:54 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt                  | 2 ++
 .../org/apache/hadoop/hdfs/server/datanode/BlockScanner.java | 2 +-
 .../apache/hadoop/hdfs/server/datanode/VolumeScanner.java    | 8 ++++----
 3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dca164/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0c56f2b..eb006eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -985,6 +985,8 @@ Release 2.7.2 - UNRELEASED
 
   IMPROVEMENTS
 
+  HDFS-8659. Block scanner INFO message is spamming logs. (Yongjun Zhang)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dca164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index 9c4dd10..be6aa83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -317,7 +317,7 @@ public class BlockScanner {
    */
   synchronized void markSuspectBlock(String storageId, ExtendedBlock block) {
     if (!isEnabled()) {
-      LOG.info("Not scanning suspicious block {} on {}, because the block " +
+      LOG.debug("Not scanning suspicious block {} on {}, because the block " +
           "scanner is disabled.", block, storageId);
       return;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dca164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 615abe9..ff655c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -656,24 +656,24 @@ public class VolumeScanner extends Thread {
 
   public synchronized void markSuspectBlock(ExtendedBlock block) {
     if (stopping) {
-      LOG.info("{}: Not scheduling suspect block {} for " +
+      LOG.debug("{}: Not scheduling suspect block {} for " +
           "rescanning, because this volume scanner is stopping.", this, block);
       return;
     }
     Boolean recent = recentSuspectBlocks.getIfPresent(block);
     if (recent != null) {
-      LOG.info("{}: Not scheduling suspect block {} for " +
+      LOG.debug("{}: Not scheduling suspect block {} for " +
           "rescanning, because we rescanned it recently.", this, block);
       return;
     }
     if (suspectBlocks.contains(block)) {
-      LOG.info("{}: suspect block {} is already queued for " +
+      LOG.debug("{}: suspect block {} is already queued for " +
           "rescanning.", this, block);
       return;
     }
     suspectBlocks.add(block);
     recentSuspectBlocks.put(block, true);
-    LOG.info("{}: Scheduling suspect block {} for rescanning.", this, block);
+    LOG.debug("{}: Scheduling suspect block {} for rescanning.", this, block);
     notify(); // wake scanner thread.
   }
 


[40/48] hadoop git commit: HDFS-8709. Clarify automatic sync in FSEditLog#logEdit.

Posted by zj...@apache.org.
HDFS-8709. Clarify automatic sync in FSEditLog#logEdit.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0823b936
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0823b936
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0823b936

Branch: refs/heads/YARN-2928
Commit: 0823b9365e00c2341714e8f488ec9112f0e72eb7
Parents: 825dbb8
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Jul 2 10:26:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  2 ++
 .../hadoop/hdfs/server/namenode/FSEditLog.java  | 28 +++++++++++++-------
 2 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0823b936/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7b96c56..6678a3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -699,6 +699,8 @@ Release 2.8.0 - UNRELEASED
     HDFS-8703. Merge refactor of DFSInputStream from ErasureCoding branch
     (vinayakumarb)
 
+    HDFS-8709. Clarify automatic sync in FSEditLog#logEdit. (wang)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0823b936/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 1b0b572..939e841 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -409,10 +409,14 @@ public class FSEditLog implements LogsPurgeable {
   }
 
   /**
-   * Write an operation to the edit log. Do not sync to persistent
-   * store yet.
+   * Write an operation to the edit log.
+   * <p/>
+   * Additionally, this will sync the edit log if required by the underlying
+   * edit stream's automatic sync policy (e.g. when the buffer is full, or
+   * if a time interval has elapsed).
    */
   void logEdit(final FSEditLogOp op) {
+    boolean needsSync = false;
     synchronized (this) {
       assert isOpenForWrite() :
         "bad state: " + state;
@@ -434,14 +438,16 @@ public class FSEditLog implements LogsPurgeable {
       endTransaction(start);
       
       // check if it is time to schedule an automatic sync
-      if (!shouldForceSync()) {
-        return;
+      needsSync = shouldForceSync();
+      if (needsSync) {
+        isAutoSyncScheduled = true;
       }
-      isAutoSyncScheduled = true;
     }
     
-    // sync buffered edit log entries to persistent store
-    logSync();
+    // Sync the log if an automatic sync is required.
+    if (needsSync) {
+      logSync();
+    }
   }
 
   /**
@@ -1191,11 +1197,13 @@ public class FSEditLog implements LogsPurgeable {
       throws IOException {
     return journalSet.getEditLogManifest(fromTxId);
   }
- 
+
   /**
    * Finalizes the current edit log and opens a new log segment.
-   * @return the transaction id of the BEGIN_LOG_SEGMENT transaction
-   * in the new log.
+   *
+   * @param layoutVersion The layout version of the new edit log segment.
+   * @return the transaction id of the BEGIN_LOG_SEGMENT transaction in the new
+   * log.
    */
   synchronized long rollEditLog(int layoutVersion) throws IOException {
     LOG.info("Rolling edit logs");


[23/48] hadoop git commit: MAPREDUCE-6407. Migrate MAPREDUCE native build to new CMake framework (Alan Burlison via Colin P. McCabe)

Posted by zj...@apache.org.
MAPREDUCE-6407. Migrate MAPREDUCE native build to new CMake framework (Alan Burlison via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a07c6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a07c6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a07c6e

Branch: refs/heads/YARN-2928
Commit: 71a07c6e1685a9bd1a4fb2fb6230b3201d9b7b76
Parents: 5b40228
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:06:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |   3 +
 .../src/CMakeLists.txt                          | 337 ++++++++-----------
 .../src/JNIFlags.cmake                          | 118 -------
 3 files changed, 137 insertions(+), 321 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a07c6e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 974af61..5f4b3ef 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -139,6 +139,9 @@ Trunk (Unreleased)
     MAPREDUCE-2632. Avoid calling the partitioner when the numReduceTasks is 1.
     (Ravi Teja Ch N V and Sunil G via kasha)
 
+    MAPREDUCE-6407. Migrate MAPREDUCE nativetask build to new CMake framework
+    (Alan Burlison via Colin P. McCabe)
+
   BUG FIXES
 
     MAPREDUCE-6191. Improve clearing stale state of Java serialization

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a07c6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 80036e8..f878a94 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -18,236 +18,170 @@
 
 cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
-
-include(JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
-    add_library(${LIBNAME} SHARED ${ARGN})
-    add_library(${LIBNAME}_static STATIC ${ARGN})
-    set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
-    target_link_libraries(${LIBNAME} ${ARGN})
-    target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
-    output_directory(${TGT} "${DIR}")
-    output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
-
-#
-# This macro alters the behavior of find_package and find_library.
-# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
-# You should save that variable before calling this function and restore it
-# after you have accomplished your goal.
-#
-# The behavior is altered in two ways:
-# 1. We always find shared libraries, never static;
-# 2. We find shared libraries with the given version number.
-#
-# On Windows this function is a no-op.  Windows does not encode
-# version number information information into library path names.
-#
-macro(set_find_shared_library_version LVERS)
-    IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-        # Mac OS uses .dylib
-        SET(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib")
-    ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
-        # FreeBSD has always .so installed.
-        SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
-    ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
-        # Windows doesn't support finding shared libraries by version.
-    ELSE()
-        # Most UNIX variants use .so
-        SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}")
-    ENDIF()
-endmacro(set_find_shared_library_version LVERS)
-
-if (NOT GENERATED_JAVAH)
-    #Must identify where the generated headers have been placed
-    MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-find_package(JNI REQUIRED)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-# primitive configs
-set(PRFLAGS "-DSIMPLE_MEMCPY")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -fno-strict-aliasing -Wall -Wno-sign-compare")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
-    -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC")
-set(D main/native/)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-INCLUDE(CheckFunctionExists)
-INCLUDE(CheckCSourceCompiles)
-#INCLUDE(CheckLibraryExists)
-INCLUDE(CheckIncludeFiles)
-#CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
-#CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
-#CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
-CHECK_INCLUDE_FILES(fcntl.h HAVE_FCNTL_H)
-CHECK_INCLUDE_FILES(malloc.h HAVE_MALLOC_H)
-CHECK_INCLUDE_FILES(mach/mach.h HAVE_MACH_MACH_H)
-CHECK_INCLUDE_FILES(memory.h HAVE_MEMORY_H)
-CHECK_INCLUDE_FILES(stddef.h HAVE_STDDEF_H)
-CHECK_INCLUDE_FILES(stdint.h HAVE_STDINT_H)
-CHECK_INCLUDE_FILES(stdlib.h HAVE_STDLIB_H)
-CHECK_INCLUDE_FILES(string.h HAVE_STRING_H)
-CHECK_INCLUDE_FILES(unistd.h HAVE_UNITSTD_H)
-CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
-CHECK_FUNCTION_EXISTS(localtime_r HAVE_LOCALTIME_R)
-CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET)
-CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR)
-CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-common/)
+include(HadoopCommon)
+
+# Add extra compiler and linker flags.
+# -Wno-sign-compare
+hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing")
+
+# Source location.
+set(SRC main/native)
+
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+    message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
+
+# Probe for headers and functions.
+include(CheckFunctionExists)
+include(CheckIncludeFiles)
+
+check_include_files(fcntl.h HAVE_FCNTL_H)
+check_include_files(malloc.h HAVE_MALLOC_H)
+check_include_files(mach/mach.h HAVE_MACH_MACH_H)
+check_include_files(memory.h HAVE_MEMORY_H)
+check_include_files(stddef.h HAVE_STDDEF_H)
+check_include_files(stdint.h HAVE_STDINT_H)
+check_include_files(stdlib.h HAVE_STDLIB_H)
+Check_include_files(string.h HAVE_STRING_H)
+check_include_files(unistd.h HAVE_UNITSTD_H)
+check_function_exists(clock_gettime HAVE_CLOCK_GETTIME)
+check_function_exists(localtime_r HAVE_LOCALTIME_R)
+check_function_exists(memset HAVE_MEMSET)
+check_function_exists(strchr HAVE_STRCHR)
+check_function_exists(strtoul HAVE_STRTOUL)
+
+# Require snappy.
+set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+hadoop_set_find_shared_library_version("1")
 find_library(SNAPPY_LIBRARY
     NAMES snappy
     PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
           ${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+set(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
 find_path(SNAPPY_INCLUDE_DIR
     NAMES snappy.h
     PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
           ${CUSTOM_SNAPPY_INCLUDE})
-if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
     GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME)
     set(SNAPPY_SOURCE_FILES
-        "${D}/src/codec/SnappyCodec.cc")
-else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+        "${SRC}/src/codec/SnappyCodec.cc")
+    set(REQUIRE_SNAPPY ${REQUIRE_SNAPPY}) # Stop warning about unused variable.
+    message(STATUS "Found Snappy: ${SNAPPY_LIBRARY}")
+else()
     set(SNAPPY_LIBRARY "")
     set(SNAPPY_INCLUDE_DIR "")
     set(SNAPPY_SOURCE_FILES "")
-    IF(REQUIRE_SNAPPY)
-        MESSAGE(FATAL_ERROR "Required snappy library could not be found.  SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
-    ENDIF(REQUIRE_SNAPPY)
-endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+    if(REQUIRE_SNAPPY)
+        message(FATAL_ERROR "Required snappy library could not be found.  SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
+    endif()
+endif()
 
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
 
 include_directories(
     ${GENERATED_JAVAH}
-    ${D}/src
-    ${D}/src/util
-    ${D}/src/lib
-    ${D}/test
+    ${SRC}/src
+    ${SRC}/src/util
+    ${SRC}/src/lib
+    ${SRC}/test
     ${CMAKE_CURRENT_SOURCE_DIR}
     ${CMAKE_BINARY_DIR}
     ${JNI_INCLUDE_DIRS}
     ${SNAPPY_INCLUDE_DIR}
 )
 # add gtest as system library to suppress gcc warnings
-include_directories(SYSTEM ${D}/gtest/include)
-
+include_directories(SYSTEM ${SRC}/gtest/include)
 
-SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
     # macosx does not have -lrt
     set(NT_DEPEND_LIBRARY dl pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
     set(SYSTEM_MAC TRUE)
-else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+else()
     set(NT_DEPEND_LIBRARY dl rt pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
     set(SYSTEM_MAC FALSE)
-endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+endif()
 
 configure_file(main/native/test.sh test/test.sh)
 
-add_dual_library(nativetask
+hadoop_add_dual_library(nativetask
     ${CMAKE_BINARY_DIR}/lz4.c
-    ${D}/src/codec/BlockCodec.cc
-    ${D}/src/codec/GzipCodec.cc
-    ${D}/src/codec/Lz4Codec.cc
+    ${SRC}/src/codec/BlockCodec.cc
+    ${SRC}/src/codec/GzipCodec.cc
+    ${SRC}/src/codec/Lz4Codec.cc
     ${SNAPPY_SOURCE_FILES}
-    ${D}/src/handler/BatchHandler.cc
-    ${D}/src/handler/MCollectorOutputHandler.cc
-    ${D}/src/handler/AbstractMapHandler.cc
-    ${D}/src/handler/CombineHandler.cc
-    ${D}/src/lib/Buffers.cc
-    ${D}/src/lib/BufferStream.cc
-    ${D}/src/lib/Compressions.cc
-    ${D}/src/lib/PartitionBucket.cc
-    ${D}/src/lib/PartitionBucketIterator.cc
-    ${D}/src/lib/FileSystem.cc
-    ${D}/src/lib/IFile.cc
-    ${D}/src/lib/jniutils.cc
-    ${D}/src/lib/Log.cc
-    ${D}/src/lib/MapOutputCollector.cc
-    ${D}/src/lib/MapOutputSpec.cc
-    ${D}/src/lib/MemoryBlock.cc
-    ${D}/src/lib/Merge.cc
-    ${D}/src/lib/NativeLibrary.cc
-    ${D}/src/lib/Iterator.cc
-    ${D}/src/lib/NativeObjectFactory.cc
-    ${D}/src/lib/NativeRuntimeJniImpl.cc
-    ${D}/src/lib/NativeTask.cc
-    ${D}/src/lib/SpillInfo.cc
-    ${D}/src/lib/Path.cc
-    ${D}/src/lib/Streams.cc
-    ${D}/src/lib/TaskCounters.cc
-    ${D}/src/util/Checksum.cc
-    ${D}/src/util/Random.cc
-    ${D}/src/util/StringUtil.cc
-    ${D}/src/util/SyncUtils.cc
-    ${D}/src/util/Timer.cc
-    ${D}/src/util/WritableUtils.cc
+    ${SRC}/src/handler/BatchHandler.cc
+    ${SRC}/src/handler/MCollectorOutputHandler.cc
+    ${SRC}/src/handler/AbstractMapHandler.cc
+    ${SRC}/src/handler/CombineHandler.cc
+    ${SRC}/src/lib/Buffers.cc
+    ${SRC}/src/lib/BufferStream.cc
+    ${SRC}/src/lib/Compressions.cc
+    ${SRC}/src/lib/PartitionBucket.cc
+    ${SRC}/src/lib/PartitionBucketIterator.cc
+    ${SRC}/src/lib/FileSystem.cc
+    ${SRC}/src/lib/IFile.cc
+    ${SRC}/src/lib/jniutils.cc
+    ${SRC}/src/lib/Log.cc
+    ${SRC}/src/lib/MapOutputCollector.cc
+    ${SRC}/src/lib/MapOutputSpec.cc
+    ${SRC}/src/lib/MemoryBlock.cc
+    ${SRC}/src/lib/Merge.cc
+    ${SRC}/src/lib/NativeLibrary.cc
+    ${SRC}/src/lib/Iterator.cc
+    ${SRC}/src/lib/NativeObjectFactory.cc
+    ${SRC}/src/lib/NativeRuntimeJniImpl.cc
+    ${SRC}/src/lib/NativeTask.cc
+    ${SRC}/src/lib/SpillInfo.cc
+    ${SRC}/src/lib/Path.cc
+    ${SRC}/src/lib/Streams.cc
+    ${SRC}/src/lib/TaskCounters.cc
+    ${SRC}/src/util/Checksum.cc
+    ${SRC}/src/util/Random.cc
+    ${SRC}/src/util/StringUtil.cc
+    ${SRC}/src/util/SyncUtils.cc
+    ${SRC}/src/util/Timer.cc
+    ${SRC}/src/util/WritableUtils.cc
 )
 
 target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
 
-add_library(gtest ${D}/gtest/gtest-all.cc)
+add_library(gtest ${SRC}/gtest/gtest-all.cc)
 set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
 add_executable(nttest
-    ${D}/test/lib/TestByteArray.cc
-    ${D}/test/lib/TestByteBuffer.cc
-    ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
-    ${D}/test/lib/TestComparatorForStdSort.cc
-    ${D}/test/lib/TestFixSizeContainer.cc
-    ${D}/test/lib/TestMemoryPool.cc
-    ${D}/test/lib/TestIterator.cc
-    ${D}/test/lib/TestKVBuffer.cc
-    ${D}/test/lib/TestMemBlockIterator.cc
-    ${D}/test/lib/TestMemoryBlock.cc
-    ${D}/test/lib/TestPartitionBucket.cc
-    ${D}/test/lib/TestReadBuffer.cc
-    ${D}/test/lib/TestReadWriteBuffer.cc
-    ${D}/test/util/TestChecksum.cc
-    ${D}/test/util/TestStringUtil.cc
-    ${D}/test/util/TestWritableUtils.cc
-    ${D}/test/TestCommand.cc
-    ${D}/test/TestConfig.cc
-    ${D}/test/TestCounter.cc
-    ${D}/test/TestCompressions.cc
-    ${D}/test/TestFileSystem.cc
-    ${D}/test/TestIFile.cc
-    ${D}/test/TestPrimitives.cc
-    ${D}/test/TestSort.cc
-    ${D}/test/TestMain.cc
-    ${D}/test/test_commons.cc)
-
+    ${SRC}/test/lib/TestByteArray.cc
+    ${SRC}/test/lib/TestByteBuffer.cc
+    ${SRC}/test/lib/TestComparatorForDualPivotQuickSort.cc
+    ${SRC}/test/lib/TestComparatorForStdSort.cc
+    ${SRC}/test/lib/TestFixSizeContainer.cc
+    ${SRC}/test/lib/TestMemoryPool.cc
+    ${SRC}/test/lib/TestIterator.cc
+    ${SRC}/test/lib/TestKVBuffer.cc
+    ${SRC}/test/lib/TestMemBlockIterator.cc
+    ${SRC}/test/lib/TestMemoryBlock.cc
+    ${SRC}/test/lib/TestPartitionBucket.cc
+    ${SRC}/test/lib/TestReadBuffer.cc
+    ${SRC}/test/lib/TestReadWriteBuffer.cc
+    ${SRC}/test/util/TestChecksum.cc
+    ${SRC}/test/util/TestStringUtil.cc
+    ${SRC}/test/util/TestWritableUtils.cc
+    ${SRC}/test/TestCommand.cc
+    ${SRC}/test/TestConfig.cc
+    ${SRC}/test/TestCounter.cc
+    ${SRC}/test/TestCompressions.cc
+    ${SRC}/test/TestFileSystem.cc
+    ${SRC}/test/TestIFile.cc
+    ${SRC}/test/TestPrimitives.cc
+    ${SRC}/test/TestSort.cc
+    ${SRC}/test/TestMain.cc
+    ${SRC}/test/test_commons.cc)
 
 target_link_libraries(nttest
      nativetask_static
@@ -255,17 +189,14 @@ target_link_libraries(nttest
      ${NT_DEPEND_LIBRARY}
 )
 
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
-    #
-    # By embedding '$ORIGIN' into the RPATH of libnativetask.so,
-    # dlopen will look in the directory containing libnativetask.so.
-    # However, $ORIGIN is not supported by all operating systems.
-    #
-    SET_TARGET_PROPERTIES(nativetask
-        PROPERTIES INSTALL_RPATH "\$ORIGIN/")
-ENDIF()
-
-SET(LIBNATIVETASK_VERSION "1.0.0")
-SET_TARGET_PROPERTIES(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
-dual_output_directory(nativetask target/usr/local/lib)
-output_directory(nttest test)
+# By embedding '$ORIGIN' into the RPATH of libnativetask.so, dlopen will look in
+# the directory containing libnativetask.so. However, $ORIGIN is not supported by
+# all operating systems.
+if(CMAKE_SYSTEM_NAME MATCHES "Linux|SunOS")
+    set_target_properties(nativetask PROPERTIES INSTALL_RPATH "\$ORIGIN/")
+endif()
+
+set(LIBNATIVETASK_VERSION "1.0.0")
+set_target_properties(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
+hadoop_dual_output_directory(nativetask target/usr/local/lib)
+hadoop_output_directory(nttest test)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a07c6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
deleted file mode 100644
index 0100b06..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-
-# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
-# This variable is set by maven.
-if (JVM_ARCH_DATA_MODEL EQUAL 32)
-    # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64
-    if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
-        set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
-    endif ()
-    if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
-        # Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use
-        # the 32-bit version of libjvm.so.
-        set(CMAKE_SYSTEM_PROCESSOR "i686")
-    endif ()
-endif (JVM_ARCH_DATA_MODEL EQUAL 32)
-
-# Determine float ABI of JVM on ARM Linux
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
-    find_program(READELF readelf)
-    if (READELF MATCHES "NOTFOUND")
-        message(WARNING "readelf not found; JVM float ABI detection disabled")
-    else (READELF MATCHES "NOTFOUND")
-        execute_process(
-            COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY}
-            OUTPUT_VARIABLE JVM_ELF_ARCH
-            ERROR_QUIET)
-        if (NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers")
-            message("Soft-float JVM detected")
-
-            # Test compilation with -mfloat-abi=softfp using an arbitrary libc function
-            # (typically fails with "fatal error: bits/predefs.h: No such file or directory"
-            # if soft-float dev libraries are not installed)
-            include(CMakePushCheckState)
-            cmake_push_check_state()
-            set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -mfloat-abi=softfp")
-            include(CheckSymbolExists)
-            check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE)
-            if (NOT SOFTFP_AVAILABLE)
-                message(FATAL_ERROR "Soft-float dev libraries required (e.g. 'apt-get install libc6-dev-armel' on Debian/Ubuntu)")
-            endif (NOT SOFTFP_AVAILABLE)
-            cmake_pop_check_state()
-
-            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfloat-abi=softfp")
-        endif ()
-    endif (READELF MATCHES "NOTFOUND")
-endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
-
-IF("${CMAKE_SYSTEM}" MATCHES "Linux")
-    #
-    # Locate JNI_INCLUDE_DIRS and JNI_LIBRARIES.
-    # Since we were invoked from Maven, we know that the JAVA_HOME environment
-    # variable is valid.  So we ignore system paths here and just use JAVA_HOME.
-    #
-    FILE(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME)
-    IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
-        SET(_java_libarch "i386")
-    ELSEIF (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
-        SET(_java_libarch "amd64")
-    ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
-        SET(_java_libarch "arm")
-    ELSE()
-        SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
-    ENDIF()
-    SET(_JDK_DIRS "${_JAVA_HOME}/jre/lib/${_java_libarch}/*"
-                  "${_JAVA_HOME}/jre/lib/${_java_libarch}"
-                  "${_JAVA_HOME}/jre/lib/*"
-                  "${_JAVA_HOME}/jre/lib"
-                  "${_JAVA_HOME}/lib/*"
-                  "${_JAVA_HOME}/lib"
-                  "${_JAVA_HOME}/include/*"
-                  "${_JAVA_HOME}/include"
-                  "${_JAVA_HOME}"
-    )
-    FIND_PATH(JAVA_INCLUDE_PATH
-        NAMES jni.h 
-        PATHS ${_JDK_DIRS}
-        NO_DEFAULT_PATH)
-    #In IBM java, it's jniport.h instead of jni_md.h
-    FIND_PATH(JAVA_INCLUDE_PATH2 
-        NAMES jni_md.h jniport.h
-        PATHS ${_JDK_DIRS}
-        NO_DEFAULT_PATH)
-    SET(JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2})
-    FIND_LIBRARY(JAVA_JVM_LIBRARY
-        NAMES rt jvm 
-        PATHS ${_JDK_DIRS}
-        NO_DEFAULT_PATH)
-    SET(JNI_LIBRARIES ${JAVA_JVM_LIBRARY})
-    MESSAGE("JAVA_HOME=${JAVA_HOME}, JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}")
-    MESSAGE("JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}, JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}")
-    IF(JAVA_JVM_LIBRARY AND JAVA_INCLUDE_PATH AND JAVA_INCLUDE_PATH2)
-        MESSAGE("Located all JNI components successfully.")
-    ELSE()
-        MESSAGE(FATAL_ERROR "Failed to find a viable JVM installation under JAVA_HOME.")
-    ENDIF()
-ELSE()
-    find_package(JNI REQUIRED)
-ENDIF()


[30/48] hadoop git commit: MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown. Contributed by Chang Li

Posted by zj...@apache.org.
MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown. Contributed by Chang Li


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7e62266
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7e62266
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7e62266

Branch: refs/heads/YARN-2928
Commit: d7e622663c19c7d5103d17e04a02cad07662cc7a
Parents: ef7b82b
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 18:01:22 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt                 |  3 +++
 .../apache/hadoop/mapreduce/v2/app/MRAppMaster.java  | 14 +++++++++++---
 .../org/apache/hadoop/mapreduce/v2/app/MRApp.java    | 15 +++++++++++++++
 3 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7e62266/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index dd739ca..68ce49b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -551,6 +551,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be
     logged in warn/info level (Chang Li via jlowe)
 
+    MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown
+    (Chang Li via jlowe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7e62266/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 149fd5d..f1d0b68 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1220,11 +1220,15 @@ public class MRAppMaster extends CompositeService {
       startJobs();
     }
   }
-  
+
+  protected void shutdownTaskLog() {
+    TaskLog.syncLogsShutdown(logSyncer);
+  }
+
   @Override
   public void stop() {
     super.stop();
-    TaskLog.syncLogsShutdown(logSyncer);
+    shutdownTaskLog();
   }
 
   private boolean isRecoverySupported() throws IOException {
@@ -1728,10 +1732,14 @@ public class MRAppMaster extends CompositeService {
     T call(Configuration conf) throws Exception;
   }
 
+  protected void shutdownLogManager() {
+    LogManager.shutdown();
+  }
+
   @Override
   protected void serviceStop() throws Exception {
     super.serviceStop();
-    LogManager.shutdown();
+    shutdownLogManager();
   }
 
   public ClientService getClientService() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7e62266/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
index 4fe4c44..b51adf2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
@@ -804,5 +804,20 @@ public class MRApp extends MRAppMaster {
             new Text(containerToken.getService()));
     return token.decodeIdentifier();
   }
+
+  @Override
+  protected void shutdownTaskLog() {
+    // Avoid closing the logging system during unit tests,
+    // otherwise subsequent MRApp instances in the same test
+    // will fail to log anything.
+  }
+
+  @Override
+  protected void shutdownLogManager() {
+    // Avoid closing the logging system during unit tests,
+    // otherwise subsequent MRApp instances in the same test
+    // will fail to log anything.
+  }
+
 }
  


[03/48] hadoop git commit: YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non network exception. Contributed by Raju Bairishetti

Posted by zj...@apache.org.
YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non network exception. Contributed by Raju Bairishetti


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f780d13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f780d13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f780d13

Branch: refs/heads/YARN-2928
Commit: 6f780d13ad9b8a71f80f6f8c5453eafc25bc1508
Parents: 38d1678
Author: Jian He <ji...@apache.org>
Authored: Mon Jun 29 13:37:32 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../apache/hadoop/yarn/client/ServerProxy.java  | 21 ++---
 .../containermanager/TestNMProxy.java           | 81 +++++++++++++-------
 3 files changed, 70 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f780d13/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0f4eef1..83b58ba 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -671,6 +671,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3860. rmadmin -transitionToActive should check the state of non-target node.
     (Masatake Iwasaki via junping_du)
 
+    YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non
+    network exception. (Raju Bairishetti via jianhe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f780d13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
index e9bcf8d..de7fc7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
@@ -53,19 +53,22 @@ public class ServerProxy {
     long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
     long retryIntervalMS =
         conf.getLong(connectRetryIntervalStr, defRetryInterval);
-    if (maxWaitTime == -1) {
-      // wait forever.
-      return RetryPolicies.RETRY_FOREVER;
-    }
 
-    Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
-        + maxWaitTimeStr + " should be a positive value.");
+    Preconditions.checkArgument((maxWaitTime == -1 || maxWaitTime > 0),
+        "Invalid Configuration. " + maxWaitTimeStr + " should be either"
+            + " positive value or -1.");
     Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
         + connectRetryIntervalStr + "should be a positive value.");
 
-    RetryPolicy retryPolicy =
-        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
-          retryIntervalMS, TimeUnit.MILLISECONDS);
+    RetryPolicy retryPolicy = null;
+    if (maxWaitTime == -1) {
+      // wait forever.
+      retryPolicy = RetryPolicies.RETRY_FOREVER;
+    } else {
+      retryPolicy =
+          RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
+              retryIntervalMS, TimeUnit.MILLISECONDS);
+    }
 
     Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
         new HashMap<Class<? extends Exception>, RetryPolicy>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f780d13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index 0b372be..102c9c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.io.retry.UnreliableInterface;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
@@ -58,8 +59,8 @@ public class TestNMProxy extends BaseContainerManagerTest {
 
   @Before
   public void setUp() throws Exception {
-    conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
-    conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
+    containerManager.start();
+    containerManager.setBlockNewContainerRequests(false);
   }
 
   @Override
@@ -77,7 +78,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
             // This causes super to throw an NMNotYetReadyException
             containerManager.setBlockNewContainerRequests(true);
           } else {
-            throw new java.net.ConnectException("start container exception");
+            if (isRetryPolicyRetryForEver()) {
+              // Throw non network exception
+              throw new IOException(
+                  new UnreliableInterface.UnreliableException());
+            } else {
+              throw new java.net.ConnectException("start container exception");
+            }
           }
         } else {
           // This stops super from throwing an NMNotYetReadyException
@@ -86,6 +93,11 @@ public class TestNMProxy extends BaseContainerManagerTest {
         return super.startContainers(requests);
       }
 
+      private boolean isRetryPolicyRetryForEver() {
+        return conf.getLong(
+            YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 1000) == -1;
+      }
+
       @Override
       public StopContainersResponse stopContainers(
           StopContainersRequest requests) throws YarnException, IOException {
@@ -110,30 +122,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
   }
 
   @Test(timeout = 20000)
-  public void testNMProxyRetry() throws Exception {
-    containerManager.start();
-    containerManager.setBlockNewContainerRequests(false);
-    StartContainersRequest allRequests =
-        Records.newRecord(StartContainersRequest.class);
-    ApplicationId appId = ApplicationId.newInstance(1, 1);
-    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+   public void testNMProxyRetry() throws Exception {
+     conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
+     conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
+     StartContainersRequest allRequests =
+         Records.newRecord(StartContainersRequest.class);
 
-    org.apache.hadoop.yarn.api.records.Token nmToken =
-        context.getNMTokenSecretManager().createNMToken(attemptId,
-          context.getNodeId(), user);
-    final InetSocketAddress address =
-        conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
-          YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
-          YarnConfiguration.DEFAULT_NM_PORT);
-    Token<NMTokenIdentifier> token =
-        ConverterUtils.convertFromYarn(nmToken,
-          SecurityUtil.buildTokenService(address));
-    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
-    ugi.addToken(token);
-
-    ContainerManagementProtocol proxy =
-        NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
-          YarnRPC.create(conf), address);
+    ContainerManagementProtocol proxy = getNMProxy();
 
     retryCount = 0;
     shouldThrowNMNotYetReadyException = false;
@@ -156,4 +151,38 @@ public class TestNMProxy extends BaseContainerManagerTest {
     proxy.startContainers(allRequests);
     Assert.assertEquals(5, retryCount);
   }
+
+  @Test(timeout = 20000, expected = IOException.class)
+  public void testShouldNotRetryForeverForNonNetworkExceptionsOnNMConnections()
+      throws Exception {
+    conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, -1);
+    StartContainersRequest allRequests =
+        Records.newRecord(StartContainersRequest.class);
+
+    ContainerManagementProtocol proxy = getNMProxy();
+
+    shouldThrowNMNotYetReadyException = false;
+    retryCount = 0;
+    proxy.startContainers(allRequests);
+  }
+
+  private ContainerManagementProtocol getNMProxy() {
+    ApplicationId appId = ApplicationId.newInstance(1, 1);
+    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+
+    org.apache.hadoop.yarn.api.records.Token nmToken =
+        context.getNMTokenSecretManager().createNMToken(attemptId,
+            context.getNodeId(), user);
+    final InetSocketAddress address =
+        conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
+            YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
+            YarnConfiguration.DEFAULT_NM_PORT);
+    Token<NMTokenIdentifier> token =
+        ConverterUtils.convertFromYarn(nmToken,
+            SecurityUtil.buildTokenService(address));
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+    ugi.addToken(token);
+    return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
+        YarnRPC.create(conf), address);
+  }
 }


[09/48] hadoop git commit: HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Contributed by Brahma Reddy Battula)

Posted by zj...@apache.org.
HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Contributed by Brahma Reddy Battula)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c51682b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c51682b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c51682b2

Branch: refs/heads/YARN-2928
Commit: c51682b2967211ddaf9e9cf4d0df70e16d1312e1
Parents: 94dca16
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jun 29 14:49:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:54 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt                       | 3 +++
 .../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java         | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51682b2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eb006eb..108a6c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -977,6 +977,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8628. Update missing command option for fetchdt
     (J.Andreina via vinayakumarb)
 
+    HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Brahma
+    Reddy Battula via Arpit Agarwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51682b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 33033e7..6bb6603 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -377,7 +377,6 @@ public class DFSck extends Configured implements Tool {
     int res = -1;
     if ((args.length == 0) || ("-files".equals(args[0]))) {
       printUsage(System.err);
-      ToolRunner.printGenericCommandUsage(System.err);
     } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
       res = 0;
     } else {


[48/48] hadoop git commit: HDFS-8577. Avoid retrying to recover lease on a file which does not exist (Contributed by J.Andreina)

Posted by zj...@apache.org.
HDFS-8577. Avoid retrying to recover lease on a file which does not exist (Contributed by J.Andreina)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06daeae6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06daeae6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06daeae6

Branch: refs/heads/YARN-2928
Commit: 06daeae65ad582600cc35174f52f728ad0f70573
Parents: 513eb8b
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Jul 3 13:35:48 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt               |  3 +++
 .../java/org/apache/hadoop/hdfs/tools/DebugAdmin.java     | 10 ++++++++--
 .../java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java |  8 ++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06daeae6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6678a3e..4f184fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1002,6 +1002,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html.
     (Brahma Reddy Battula via Arpit Agarwal)
 
+    HDFS-8577. Avoid retrying to recover lease on a file which does not exist
+    (J.Andreina via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06daeae6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index 41f1ca4..d179a5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.tools;
 
 import java.io.DataInputStream;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -254,6 +255,11 @@ public class DebugAdmin extends Configured implements Tool {
         IOException ioe = null;
         try {
           recovered = dfs.recoverLease(new Path(pathStr));
+        } catch (FileNotFoundException e) {
+          System.err.println("recoverLease got exception: " + e.getMessage());
+          System.err.println("Giving up on recoverLease for " + pathStr +
+              " after 1 try");
+          return 1;
         } catch (IOException e) {
           ioe = e;
         }
@@ -262,8 +268,8 @@ public class DebugAdmin extends Configured implements Tool {
           return 0;
         }
         if (ioe != null) {
-          System.err.println("recoverLease got exception: ");
-          ioe.printStackTrace();
+          System.err.println("recoverLease got exception: " +
+              ioe.getMessage());
         } else {
           System.err.println("recoverLease returned false.");
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06daeae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
index 52b194d..07f70e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
@@ -37,6 +37,7 @@ import java.io.PrintStream;
 
 import static org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil.*;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class TestDebugAdmin {
   private MiniDFSCluster cluster;
@@ -116,4 +117,11 @@ public class TestDebugAdmin {
             "-block", blockFile.getAbsolutePath()})
     );
   }
+
+  @Test(timeout = 60000)
+  public void testRecoverLeaseforFileNotFound() throws Exception {
+    assertTrue(runCmd(new String[] {
+        "recoverLease", "-path", "/foo", "-retries", "2" }).contains(
+        "Giving up on recoverLease for /foo after 1 try"));
+  }
 }


[06/48] hadoop git commit: HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo. Contributed by Zhe Zhang.

Posted by zj...@apache.org.
HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo. Contributed by Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9d10052
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9d10052
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9d10052

Branch: refs/heads/YARN-2928
Commit: e9d100527b5a5b994d77770a5a6bf8a327d2139f
Parents: 130d7d1
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jun 29 12:12:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../CacheReplicationMonitor.java                |  4 +--
 .../blockmanagement/DatanodeDescriptor.java     | 33 ++++++++++----------
 .../server/blockmanagement/DatanodeManager.java | 26 +++++++--------
 .../blockmanagement/DatanodeStorageInfo.java    | 13 ++++++--
 5 files changed, 45 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e55f340..0c56f2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -682,6 +682,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8623. Refactor NameNode handling of invalid, corrupt, and under-recovery
     blocks. (Zhe Zhang via jing9)
 
+    HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and
+    DatanodeStorageInfo. (Zhe Zhang via wang)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index a0f3503..2f81ddf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -452,8 +452,8 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
         file.getFullPathName(), cachedTotal, neededTotal);
   }
 
-  private String findReasonForNotCaching(CachedBlock cblock, 
-          BlockInfo blockInfo) {
+  private String findReasonForNotCaching(CachedBlock cblock,
+      BlockInfo blockInfo) {
     if (blockInfo == null) {
       // Somehow, a cache report with the block arrived, but the block
       // reports from the DataNode haven't (yet?) described such a block.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index dd7b301..99def6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -64,7 +64,8 @@ public class DatanodeDescriptor extends DatanodeInfo {
 
   // Stores status of decommissioning.
   // If node is not decommissioning, do not use this object for anything.
-  public final DecommissioningStatus decommissioningStatus = new DecommissioningStatus();
+  public final DecommissioningStatus decommissioningStatus =
+      new DecommissioningStatus();
 
   private long curBlockReportId = 0;
 
@@ -115,7 +116,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
         return null;
       }
 
-      List<E> results = new ArrayList<E>();
+      List<E> results = new ArrayList<>();
       for(; !blockq.isEmpty() && numBlocks > 0; numBlocks--) {
         results.add(blockq.poll());
       }
@@ -135,7 +136,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
   }
 
   private final Map<String, DatanodeStorageInfo> storageMap = 
-      new HashMap<String, DatanodeStorageInfo>();
+      new HashMap<>();
 
   /**
    * A list of CachedBlock objects on this datanode.
@@ -217,12 +218,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
   private long bandwidth;
 
   /** A queue of blocks to be replicated by this datanode */
-  private final BlockQueue<BlockTargetPair> replicateBlocks = new BlockQueue<BlockTargetPair>();
+  private final BlockQueue<BlockTargetPair> replicateBlocks =
+      new BlockQueue<>();
   /** A queue of blocks to be recovered by this datanode */
   private final BlockQueue<BlockInfoUnderConstruction> recoverBlocks =
-                                new BlockQueue<BlockInfoUnderConstruction>();
+      new BlockQueue<>();
   /** A set of blocks to be invalidated by this datanode */
-  private final LightWeightHashSet<Block> invalidateBlocks = new LightWeightHashSet<Block>();
+  private final LightWeightHashSet<Block> invalidateBlocks =
+      new LightWeightHashSet<>();
 
   /* Variables for maintaining number of blocks scheduled to be written to
    * this storage. This count is approximate and might be slightly bigger
@@ -230,9 +233,9 @@ public class DatanodeDescriptor extends DatanodeInfo {
    * while writing the block).
    */
   private EnumCounters<StorageType> currApproxBlocksScheduled
-      = new EnumCounters<StorageType>(StorageType.class);
+      = new EnumCounters<>(StorageType.class);
   private EnumCounters<StorageType> prevApproxBlocksScheduled
-      = new EnumCounters<StorageType>(StorageType.class);
+      = new EnumCounters<>(StorageType.class);
   private long lastBlocksScheduledRollTime = 0;
   private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min
   private int volumeFailures = 0;
@@ -276,6 +279,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
       return storageMap.get(storageID);
     }
   }
+
   DatanodeStorageInfo[] getStorageInfos() {
     synchronized (storageMap) {
       final Collection<DatanodeStorageInfo> storages = storageMap.values();
@@ -321,7 +325,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
               Long.toHexString(curBlockReportId));
           iter.remove();
           if (zombies == null) {
-            zombies = new LinkedList<DatanodeStorageInfo>();
+            zombies = new LinkedList<>();
           }
           zombies.add(storageInfo);
         }
@@ -350,10 +354,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
    */
   boolean removeBlock(String storageID, BlockInfo b) {
     DatanodeStorageInfo s = getStorageInfo(storageID);
-    if (s != null) {
-      return s.removeBlock(b);
-    }
-    return false;
+    return s != null && s.removeBlock(b);
   }
 
   public void resetBlocks() {
@@ -449,7 +450,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
           + this.volumeFailures + " to " + volFailures);
       synchronized (storageMap) {
         failedStorageInfos =
-            new HashSet<DatanodeStorageInfo>(storageMap.values());
+            new HashSet<>(storageMap.values());
       }
     }
 
@@ -505,7 +506,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
       HashMap<String, DatanodeStorageInfo> excessStorages;
 
       // Init excessStorages with all known storages.
-      excessStorages = new HashMap<String, DatanodeStorageInfo>(storageMap);
+      excessStorages = new HashMap<>(storageMap);
 
       // Remove storages that the DN reported in the heartbeat.
       for (final StorageReport report : reports) {
@@ -542,7 +543,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
     private final List<Iterator<BlockInfo>> iterators;
     
     private BlockIterator(final DatanodeStorageInfo... storages) {
-      List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
+      List<Iterator<BlockInfo>> iterators = new ArrayList<>();
       for (DatanodeStorageInfo e : storages) {
         iterators.add(e.getBlockIterator());
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 8143fb4..4266004 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -85,7 +85,7 @@ public class DatanodeManager {
    * Mapping: StorageID -> DatanodeDescriptor
    */
   private final Map<String, DatanodeDescriptor> datanodeMap
-      = new HashMap<String, DatanodeDescriptor>();
+      = new HashMap<>();
 
   /** Cluster network topology */
   private final NetworkTopology networktopology;
@@ -162,7 +162,7 @@ public class DatanodeManager {
    * Software version -> Number of datanodes with this version
    */
   private HashMap<String, Integer> datanodesSoftwareVersions =
-    new HashMap<String, Integer>(4, 0.75f);
+    new HashMap<>(4, 0.75f);
   
   /**
    * The minimum time between resending caching directives to Datanodes,
@@ -217,7 +217,7 @@ public class DatanodeManager {
     // locations of those hosts in the include list and store the mapping
     // in the cache; so future calls to resolve will be fast.
     if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
-      final ArrayList<String> locations = new ArrayList<String>();
+      final ArrayList<String> locations = new ArrayList<>();
       for (InetSocketAddress addr : hostFileManager.getIncludes()) {
         locations.add(addr.getAddress().getHostAddress());
       }
@@ -370,7 +370,7 @@ public class DatanodeManager {
     // here we should get node but not datanode only .
     Node client = getDatanodeByHost(targethost);
     if (client == null) {
-      List<String> hosts = new ArrayList<String> (1);
+      List<String> hosts = new ArrayList<> (1);
       hosts.add(targethost);
       List<String> resolvedHosts = dnsToSwitchMapping.resolve(hosts);
       if (resolvedHosts != null && !resolvedHosts.isEmpty()) {
@@ -522,7 +522,7 @@ public class DatanodeManager {
   void datanodeDump(final PrintWriter out) {
     synchronized (datanodeMap) {
       Map<String,DatanodeDescriptor> sortedDatanodeMap =
-          new TreeMap<String,DatanodeDescriptor>(datanodeMap);
+          new TreeMap<>(datanodeMap);
       out.println("Metasave: Number of datanodes: " + datanodeMap.size());
       for (DatanodeDescriptor node : sortedDatanodeMap.values()) {
         out.println(node.dumpDatanode());
@@ -660,7 +660,7 @@ public class DatanodeManager {
 
   private void countSoftwareVersions() {
     synchronized(datanodeMap) {
-      HashMap<String, Integer> versionCount = new HashMap<String, Integer>();
+      HashMap<String, Integer> versionCount = new HashMap<>();
       for(DatanodeDescriptor dn: datanodeMap.values()) {
         // Check isAlive too because right after removeDatanode(),
         // isDatanodeDead() is still true 
@@ -677,7 +677,7 @@ public class DatanodeManager {
 
   public HashMap<String, Integer> getDatanodesSoftwareVersions() {
     synchronized(datanodeMap) {
-      return new HashMap<String, Integer> (this.datanodesSoftwareVersions);
+      return new HashMap<> (this.datanodesSoftwareVersions);
     }
   }
   
@@ -710,7 +710,7 @@ public class DatanodeManager {
    */
   private String resolveNetworkLocation (DatanodeID node) 
       throws UnresolvedTopologyException {
-    List<String> names = new ArrayList<String>(1);
+    List<String> names = new ArrayList<>(1);
     if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
       names.add(node.getIpAddr());
     } else {
@@ -1000,7 +1000,7 @@ public class DatanodeManager {
       // If the network location is invalid, clear the cached mappings
       // so that we have a chance to re-add this DataNode with the
       // correct network location later.
-      List<String> invalidNodeNames = new ArrayList<String>(3);
+      List<String> invalidNodeNames = new ArrayList<>(3);
       // clear cache for nodes in IP or Hostname
       invalidNodeNames.add(nodeReg.getIpAddr());
       invalidNodeNames.add(nodeReg.getHostName());
@@ -1275,7 +1275,7 @@ public class DatanodeManager {
     final HostFileManager.HostSet excludedNodes = hostFileManager.getExcludes();
 
     synchronized(datanodeMap) {
-      nodes = new ArrayList<DatanodeDescriptor>(datanodeMap.size());
+      nodes = new ArrayList<>(datanodeMap.size());
       for (DatanodeDescriptor dn : datanodeMap.values()) {
         final boolean isDead = isDatanodeDead(dn);
         final boolean isDecommissioning = dn.isDecommissionInProgress();
@@ -1351,7 +1351,7 @@ public class DatanodeManager {
       VolumeFailureSummary volumeFailureSummary) throws IOException {
     synchronized (heartbeatManager) {
       synchronized (datanodeMap) {
-        DatanodeDescriptor nodeinfo = null;
+        DatanodeDescriptor nodeinfo;
         try {
           nodeinfo = getDatanode(nodeReg);
         } catch(UnregisteredNodeException e) {
@@ -1389,7 +1389,7 @@ public class DatanodeManager {
             final DatanodeStorageInfo[] storages = b.getExpectedStorageLocations();
             // Skip stale nodes during recovery - not heart beated for some time (30s by default).
             final List<DatanodeStorageInfo> recoveryLocations =
-                new ArrayList<DatanodeStorageInfo>(storages.length);
+                new ArrayList<>(storages.length);
             for (int i = 0; i < storages.length; i++) {
               if (!storages[i].getDatanodeDescriptor().isStale(staleInterval)) {
                 recoveryLocations.add(storages[i]);
@@ -1431,7 +1431,7 @@ public class DatanodeManager {
           return new DatanodeCommand[] { brCommand };
         }
 
-        final List<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
+        final List<DatanodeCommand> cmds = new ArrayList<>();
         //check pending replication
         List<BlockTargetPair> pendingList = nodeinfo.getReplicationCommand(
               maxTransfers);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d10052/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 65b83e1..92841a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -37,8 +37,9 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport;
 public class DatanodeStorageInfo {
   public static final DatanodeStorageInfo[] EMPTY_ARRAY = {};
 
-  public static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) {
-    return toDatanodeInfos(Arrays.asList(storages));
+  public static DatanodeInfo[] toDatanodeInfos(
+      DatanodeStorageInfo[] storages) {
+    return storages == null ? null: toDatanodeInfos(Arrays.asList(storages));
   }
   static DatanodeInfo[] toDatanodeInfos(List<DatanodeStorageInfo> storages) {
     final DatanodeInfo[] datanodes = new DatanodeInfo[storages.size()];
@@ -58,6 +59,9 @@ public class DatanodeStorageInfo {
   }
 
   public static String[] toStorageIDs(DatanodeStorageInfo[] storages) {
+    if (storages == null) {
+      return null;
+    }
     String[] storageIDs = new String[storages.length];
     for(int i = 0; i < storageIDs.length; i++) {
       storageIDs[i] = storages[i].getStorageID();
@@ -66,6 +70,9 @@ public class DatanodeStorageInfo {
   }
 
   public static StorageType[] toStorageTypes(DatanodeStorageInfo[] storages) {
+    if (storages == null) {
+      return null;
+    }
     StorageType[] storageTypes = new StorageType[storages.length];
     for(int i = 0; i < storageTypes.length; i++) {
       storageTypes[i] = storages[i].getStorageType();
@@ -380,6 +387,6 @@ public class DatanodeStorageInfo {
   }
 
   static enum AddBlockResult {
-    ADDED, REPLACED, ALREADY_EXIST;
+    ADDED, REPLACED, ALREADY_EXIST
   }
 }


[32/48] hadoop git commit: YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null attempt. Contributed by nijel.

Posted by zj...@apache.org.
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a
null attempt. Contributed by nijel.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14bfd5d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14bfd5d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14bfd5d3

Branch: refs/heads/YARN-2928
Commit: 14bfd5d3c637f50d252fd44ef8396c8777272819
Parents: 09cd3df
Author: Devaraj K <de...@apache.org>
Authored: Wed Jul 1 19:03:44 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../scheduler/AbstractYarnScheduler.java        | 34 +++++----
 .../scheduler/TestAbstractYarnScheduler.java    | 74 ++++++++++++++++++--
 3 files changed, 91 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bfd5d3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index abba1b8..fff46b8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -686,6 +686,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3823. Fix mismatch in default values for 
     yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
 
+    YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
+    attempt. (nijel via devaraj)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bfd5d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ae927f1..aad76fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.SettableFuture;
 
 
@@ -451,25 +452,30 @@ public abstract class AbstractYarnScheduler
     new Timer().schedule(new TimerTask() {
       @Override
       public void run() {
-        for (SchedulerApplication<T> app : applications.values()) {
+        clearPendingContainerCache();
+        LOG.info("Release request cache is cleaned up");
+      }
+    }, nmExpireInterval);
+  }
 
-          T attempt = app.getCurrentAppAttempt();
-          synchronized (attempt) {
-            for (ContainerId containerId : attempt.getPendingRelease()) {
-              RMAuditLogger.logFailure(
-                app.getUser(),
+  @VisibleForTesting
+  public void clearPendingContainerCache() {
+    for (SchedulerApplication<T> app : applications.values()) {
+      T attempt = app.getCurrentAppAttempt();
+      if (attempt != null) {
+        synchronized (attempt) {
+          for (ContainerId containerId : attempt.getPendingRelease()) {
+            RMAuditLogger.logFailure(app.getUser(),
                 AuditConstants.RELEASE_CONTAINER,
-                "Unauthorized access or invalid container",
-                "Scheduler",
-                "Trying to release container not owned by app or with invalid id.",
-                attempt.getApplicationId(), containerId);
-            }
-            attempt.getPendingRelease().clear();
+                "Unauthorized access or invalid container", "Scheduler",
+                "Trying to release container not owned by app "
+                    + "or with invalid id.", attempt.getApplicationId(),
+                containerId);
           }
+          attempt.getPendingRelease().clear();
         }
-        LOG.info("Release request cache is cleaned up");
       }
-    }, nmExpireInterval);
+    }
   }
 
   // clean up a completed container

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bfd5d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 48ce822..91dd249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -18,26 +18,36 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.util.HashMap;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
 @SuppressWarnings("unchecked")
 public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
 
@@ -341,6 +351,58 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
     }
   }
 
+  /*
+   * This test case is to test the pending containers are cleared from the
+   * attempt even if one of the application in the list have current attempt as
+   * null (no attempt).
+   */
+  @SuppressWarnings({ "rawtypes" })
+  @Test(timeout = 10000)
+  public void testReleasedContainerIfAppAttemptisNull() throws Exception {
+    YarnConfiguration conf=getConf();
+    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+    MemoryRMStateStore memStore = new MemoryRMStateStore();
+    memStore.init(conf);
+    MockRM rm1 = new MockRM(conf, memStore);
+    try {
+      rm1.start();
+      MockNM nm1 =
+          new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
+      nm1.registerNode();
+
+      AbstractYarnScheduler scheduler =
+          (AbstractYarnScheduler) rm1.getResourceScheduler();
+      // Mock App without attempt
+      RMApp mockAPp =
+          new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW);
+      SchedulerApplication<FiCaSchedulerApp> application =
+          new SchedulerApplication<FiCaSchedulerApp>(null, mockAPp.getUser());
+
+      // Second app with one app attempt
+      RMApp app = rm1.submitApp(200);
+      MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1);
+      final ContainerId runningContainer =
+          ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+      am1.allocate(null, Arrays.asList(runningContainer));
+
+      Map schedulerApplications = scheduler.getSchedulerApplications();
+      SchedulerApplication schedulerApp =
+          (SchedulerApplication) scheduler.getSchedulerApplications().get(
+              app.getApplicationId());
+      schedulerApplications.put(mockAPp.getApplicationId(), application);
+
+      scheduler.clearPendingContainerCache();
+
+      Assert.assertEquals("Pending containers are not released "
+          + "when one of the application attempt is null !", schedulerApp
+          .getCurrentAppAttempt().getPendingRelease().size(), 0);
+    } finally {
+      if (rm1 != null) {
+        rm1.stop();
+      }
+    }
+  }
+
   private void verifyMaximumResourceCapability(
       Resource expectedMaximumResource, AbstractYarnScheduler scheduler) {
 


[34/48] hadoop git commit: HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html. (Controibuted by Brahma Reddy Battula)

Posted by zj...@apache.org.
HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html. (Controibuted by Brahma Reddy Battula)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d57f65e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d57f65e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d57f65e6

Branch: refs/heads/YARN-2928
Commit: d57f65e6a18f2eef3c772196075c1e0e4d21b9a2
Parents: 23ac91c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jul 1 10:36:04 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:59 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt                       | 3 +++
 hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57f65e6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 65f286d..3736932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -992,6 +992,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8579. Update HDFS usage with missing options
     (J.Andreina via vinayakumarb)
 
+    HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html.
+    (Brahma Reddy Battula via Arpit Agarwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57f65e6/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 094dda1..1c2026c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -298,7 +298,7 @@ See the [HDFS Transparent Encryption Documentation](./TransparentEncryption.html
 
 ### `datanode`
 
-Usage: `hdfs datanode [-regular | -rollback | -rollingupgrace rollback]`
+Usage: `hdfs datanode [-regular | -rollback | -rollingupgrade rollback]`
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |


[39/48] hadoop git commit: YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id properly in log. Contributed by Bibin A Chundatt.

Posted by zj...@apache.org.
YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
properly in log. Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec285b32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec285b32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec285b32

Branch: refs/heads/YARN-2928
Commit: ec285b3259c8a369b88ff44724d1670633a64cc6
Parents: a8a6574
Author: Devaraj K <de...@apache.org>
Authored: Thu Jul 2 10:20:31 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                          |  3 +++
 .../resourcemanager/scheduler/fair/FSSchedulerNode.java  | 11 ++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec285b32/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3e42e61..9780b4f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -689,6 +689,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
     attempt. (nijel via devaraj)
 
+    YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
+    properly in log. (Bibin A Chundatt via devaraj)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec285b32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index be08dff..c86201a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -68,12 +68,13 @@ public class FSSchedulerNode extends SchedulerNode {
             " on node " + this);
       }
 
-      LOG.info("Updated reserved container " + 
-          container.getContainer().getId() + " on node " + 
-          this + " for application " + application);
+      LOG.info("Updated reserved container " + container.getContainer().getId()
+          + " on node " + this + " for application "
+          + application.getApplicationId());
     } else {
-      LOG.info("Reserved container " + container.getContainer().getId() + 
-          " on node " + this + " for application " + application);
+      LOG.info("Reserved container " + container.getContainer().getId()
+          + " on node " + this + " for application "
+          + application.getApplicationId());
     }
     setReservedContainer(container);
     this.reservedAppSchedulable = (FSAppAttempt) application;


[08/48] hadoop git commit: YARN-3770. SerializedException should also handle java.lang.Error on de-serialization. Contributed by Lavkesh Lahngir

Posted by zj...@apache.org.
YARN-3770. SerializedException should also handle java.lang.Error on de-serialization. Contributed by Lavkesh Lahngir


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcb992af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcb992af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcb992af

Branch: refs/heads/YARN-2928
Commit: bcb992af7ae0d0317a3cf314c75090642bbab603
Parents: 3aff2fd
Author: Jian He <ji...@apache.org>
Authored: Mon Jun 29 14:31:32 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:02 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                         |  3 +++
 .../api/records/impl/pb/SerializedExceptionPBImpl.java  |  2 +-
 .../records/impl/pb/TestSerializedExceptionPBImpl.java  | 12 ++++++++++--
 3 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcb992af/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 83b58ba..d8cfa4b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -674,6 +674,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non
     network exception. (Raju Bairishetti via jianhe)
 
+    YARN-3770. SerializedException should also handle java.lang.Error on
+    de-serialization. (Lavkesh Lahngir via jianhe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcb992af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
index fd9e170..2ec232e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
@@ -101,7 +101,7 @@ public class SerializedExceptionPBImpl extends SerializedException {
     } else if (RuntimeException.class.isAssignableFrom(realClass)) {
       classType = RuntimeException.class;
     } else {
-      classType = Exception.class;
+      classType = Throwable.class;
     }
     return instantiateException(realClass.asSubclass(classType), getMessage(),
       cause == null ? null : cause.deSerialize());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcb992af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
index ac7e40e..ecfa63e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
@@ -20,10 +20,9 @@ package org.apache.hadoop.yarn.api.records.impl.pb;
 
 import java.nio.channels.ClosedChannelException;
 
-import org.junit.Assert;
-import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestSerializedExceptionPBImpl {
@@ -79,4 +78,13 @@ public class TestSerializedExceptionPBImpl {
     SerializedExceptionPBImpl pb3 = new SerializedExceptionPBImpl();
     Assert.assertEquals(defaultProto.getTrace(), pb3.getRemoteTrace());
   }
+
+  @Test
+  public void testThrowableDeserialization() {
+    // java.lang.Error should also be serializable
+    Error ex = new Error();
+    SerializedExceptionPBImpl pb = new SerializedExceptionPBImpl();
+    pb.init(ex);
+    Assert.assertEquals(ex.getClass(), pb.deSerialize().getClass());
+  }
 }


[28/48] hadoop git commit: HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for cygwin in branch-2. Contributed by Li Lu.

Posted by zj...@apache.org.
HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for cygwin in branch-2. Contributed by Li Lu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41bde4fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41bde4fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41bde4fc

Branch: refs/heads/YARN-2928
Commit: 41bde4fcbd5c25c18eeee48f435403006d8d0d2b
Parents: 36a33eb
Author: cnauroth <cn...@apache.org>
Authored: Tue Jun 30 16:54:30 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
 1 file changed, 3 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/41bde4fc/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4e5f6bb..74d428f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -913,6 +913,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil
     and fix for HA namespaces (rchiang via rkanter)
 
+    HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for
+    cygwin in branch-2. (Li Lu via cnauroth)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES


[29/48] hadoop git commit: HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command name using reflection. (Lei Xu)

Posted by zj...@apache.org.
 HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command name using reflection. (Lei Xu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21765401
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21765401
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21765401

Branch: refs/heads/YARN-2928
Commit: 21765401e48c9eb578cd9b1f98ea7e4e385bc4fd
Parents: 41bde4f
Author: Lei Xu <le...@apache.org>
Authored: Tue Jun 30 17:26:49 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +
 .../apache/hadoop/fs/TestFsShellReturnCode.java | 60 +++++++++++++++-----
 .../org/apache/hadoop/fs/shell/TestMove.java    |  1 +
 3 files changed, 49 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/21765401/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 74d428f..39e2e5e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -916,6 +916,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for
     cygwin in branch-2. (Li Lu via cnauroth)
 
+    HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
+    name using reflection. (Lei (Eddy) Xu)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/21765401/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
index dcc19df..a2276ec 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -402,7 +403,7 @@ public class TestFsShellReturnCode {
     // processing a file throws an interrupt, it should blow on first file
     assertEquals(1, InterruptCommand.processed);
     assertEquals(130, exitCode);
-    
+
     exitCode = shell.run(
         new String[]{ "-testInterrupt", d.toString() });
     // processing a file throws an interrupt, it should blow on file
@@ -412,17 +413,32 @@ public class TestFsShellReturnCode {
   }
 
   /**
+   * Faked Chown class for {@link testChownUserAndGroupValidity()}.
+   *
+   * The test only covers argument parsing, so override to skip processing.
+   */
+  private static class FakeChown extends FsShellPermissions.Chown {
+    public static String NAME = "chown";
+    @Override
+    protected void processArgument(PathData item) {
+    }
+  }
+
+  /**
    * Tests combinations of valid and invalid user and group arguments to chown.
    */
   @Test
   public void testChownUserAndGroupValidity() {
-    // This test only covers argument parsing, so override to skip processing.
-    FsCommand chown = new FsShellPermissions.Chown() {
-      @Override
-      protected void processArgument(PathData item) {
-      }
-    };
-    chown.setConf(new Configuration());
+    testChownUserAndGroupValidity(true);
+    testChownUserAndGroupValidity(false);
+  }
+
+  private void testChownUserAndGroupValidity(boolean enableWarning) {
+    Configuration conf = new Configuration();
+    conf.setBoolean(
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
+    FsCommand chown = new FakeChown();
+    chown.setConf(conf);
 
     // The following are valid (no exception expected).
     chown.run("user", "/path");
@@ -447,17 +463,31 @@ public class TestFsShellReturnCode {
   }
 
   /**
+   * Faked Chgrp class for {@link testChgrpGroupValidity()}.
+   * The test only covers argument parsing, so override to skip processing.
+   */
+  private static class FakeChgrp extends FsShellPermissions.Chgrp {
+    public static String NAME = "chgrp";
+    @Override
+    protected void processArgument(PathData item) {
+    }
+  }
+
+  /**
    * Tests valid and invalid group arguments to chgrp.
    */
   @Test
   public void testChgrpGroupValidity() {
-    // This test only covers argument parsing, so override to skip processing.
-    FsCommand chgrp = new FsShellPermissions.Chgrp() {
-      @Override
-      protected void processArgument(PathData item) {
-      }
-    };
-    chgrp.setConf(new Configuration());
+    testChgrpGroupValidity(true);
+    testChgrpGroupValidity(false);
+  }
+
+  private void testChgrpGroupValidity(boolean enableWarning) {
+    Configuration conf = new Configuration();
+    conf.setBoolean(
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
+    FsShellPermissions.Chgrp chgrp = new FakeChgrp();
+    chgrp.setConf(conf);
 
     // The following are valid (no exception expected).
     chgrp.run("group", "/path");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/21765401/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
index 6599edf..94930e5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
@@ -114,6 +114,7 @@ public class TestMove {
   }
     
   private static class InstrumentedRenameCommand extends MoveCommands.Rename {
+    public static String NAME = "InstrumentedRename";
     private Exception error = null;
     @Override
     public void displayError(Exception e) {


[07/48] hadoop git commit: HADOOP-12107. long running apps may have a huge number of StatisticsData instances under FileSystem (Sangjin Lee via Ming Ma)

Posted by zj...@apache.org.
HADOOP-12107. long running apps may have a huge number of StatisticsData instances under FileSystem (Sangjin Lee via Ming Ma)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/296d1507
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/296d1507
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/296d1507

Branch: refs/heads/YARN-2928
Commit: 296d1507bea76acf37537e408f295c6f66c4f7d7
Parents: bcb992a
Author: Ming Ma <mi...@apache.org>
Authored: Mon Jun 29 14:37:38 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:02 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../java/org/apache/hadoop/fs/FileSystem.java   | 140 +++++++++++++------
 .../apache/hadoop/fs/FCStatisticsBaseTest.java  |  56 +++++++-
 3 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/296d1507/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a9b44e3..50192ae 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -490,6 +490,9 @@ Trunk (Unreleased)
     HADOOP-11347. RawLocalFileSystem#mkdir and create should honor umask (Varun
     Saxena via Colin P. McCabe)
 
+    HADOOP-12107. long running apps may have a huge number of StatisticsData
+    instances under FileSystem (Sangjin Lee via Ming Ma)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/296d1507/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 3f9e3bd..1d7bc87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.fs;
 import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.lang.ref.WeakReference;
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.PrivilegedExceptionAction;
@@ -32,7 +33,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
@@ -2920,16 +2920,6 @@ public abstract class FileSystem extends Configured implements Closeable {
       volatile int readOps;
       volatile int largeReadOps;
       volatile int writeOps;
-      /**
-       * Stores a weak reference to the thread owning this StatisticsData.
-       * This allows us to remove StatisticsData objects that pertain to
-       * threads that no longer exist.
-       */
-      final WeakReference<Thread> owner;
-
-      StatisticsData(WeakReference<Thread> owner) {
-        this.owner = owner;
-      }
 
       /**
        * Add another StatisticsData object to this one.
@@ -3000,17 +2990,37 @@ public abstract class FileSystem extends Configured implements Closeable {
      * Thread-local data.
      */
     private final ThreadLocal<StatisticsData> threadData;
-    
+
     /**
-     * List of all thread-local data areas.  Protected by the Statistics lock.
+     * Set of all thread-local data areas.  Protected by the Statistics lock.
+     * The references to the statistics data are kept using phantom references
+     * to the associated threads. Proper clean-up is performed by the cleaner
+     * thread when the threads are garbage collected.
      */
-    private LinkedList<StatisticsData> allData;
+    private final Set<StatisticsDataReference> allData;
+
+    /**
+     * Global reference queue and a cleaner thread that manage statistics data
+     * references from all filesystem instances.
+     */
+    private static final ReferenceQueue<Thread> STATS_DATA_REF_QUEUE;
+    private static final Thread STATS_DATA_CLEANER;
+
+    static {
+      STATS_DATA_REF_QUEUE = new ReferenceQueue<Thread>();
+      // start a single daemon cleaner thread
+      STATS_DATA_CLEANER = new Thread(new StatisticsDataReferenceCleaner());
+      STATS_DATA_CLEANER.
+          setName(StatisticsDataReferenceCleaner.class.getName());
+      STATS_DATA_CLEANER.setDaemon(true);
+      STATS_DATA_CLEANER.start();
+    }
 
     public Statistics(String scheme) {
       this.scheme = scheme;
-      this.rootData = new StatisticsData(null);
+      this.rootData = new StatisticsData();
       this.threadData = new ThreadLocal<StatisticsData>();
-      this.allData = null;
+      this.allData = new HashSet<StatisticsDataReference>();
     }
 
     /**
@@ -3020,7 +3030,7 @@ public abstract class FileSystem extends Configured implements Closeable {
      */
     public Statistics(Statistics other) {
       this.scheme = other.scheme;
-      this.rootData = new StatisticsData(null);
+      this.rootData = new StatisticsData();
       other.visitAll(new StatisticsAggregator<Void>() {
         @Override
         public void accept(StatisticsData data) {
@@ -3032,6 +3042,63 @@ public abstract class FileSystem extends Configured implements Closeable {
         }
       });
       this.threadData = new ThreadLocal<StatisticsData>();
+      this.allData = new HashSet<StatisticsDataReference>();
+    }
+
+    /**
+     * A phantom reference to a thread that also includes the data associated
+     * with that thread. On the thread being garbage collected, it is enqueued
+     * to the reference queue for clean-up.
+     */
+    private class StatisticsDataReference extends PhantomReference<Thread> {
+      private final StatisticsData data;
+
+      public StatisticsDataReference(StatisticsData data, Thread thread) {
+        super(thread, STATS_DATA_REF_QUEUE);
+        this.data = data;
+      }
+
+      public StatisticsData getData() {
+        return data;
+      }
+
+      /**
+       * Performs clean-up action when the associated thread is garbage
+       * collected.
+       */
+      public void cleanUp() {
+        // use the statistics lock for safety
+        synchronized (Statistics.this) {
+          /*
+           * If the thread that created this thread-local data no longer exists,
+           * remove the StatisticsData from our list and fold the values into
+           * rootData.
+           */
+          rootData.add(data);
+          allData.remove(this);
+        }
+      }
+    }
+
+    /**
+     * Background action to act on references being removed.
+     */
+    private static class StatisticsDataReferenceCleaner implements Runnable {
+      @Override
+      public void run() {
+        while (true) {
+          try {
+            StatisticsDataReference ref =
+                (StatisticsDataReference)STATS_DATA_REF_QUEUE.remove();
+            ref.cleanUp();
+          } catch (Throwable th) {
+            // the cleaner thread should continue to run even if there are
+            // exceptions, including InterruptedException
+            LOG.warn("exception in the cleaner thread but it will continue to "
+                + "run", th);
+          }
+        }
+      }
     }
 
     /**
@@ -3040,14 +3107,12 @@ public abstract class FileSystem extends Configured implements Closeable {
     public StatisticsData getThreadStatistics() {
       StatisticsData data = threadData.get();
       if (data == null) {
-        data = new StatisticsData(
-            new WeakReference<Thread>(Thread.currentThread()));
+        data = new StatisticsData();
         threadData.set(data);
+        StatisticsDataReference ref =
+            new StatisticsDataReference(data, Thread.currentThread());
         synchronized(this) {
-          if (allData == null) {
-            allData = new LinkedList<StatisticsData>();
-          }
-          allData.add(data);
+          allData.add(ref);
         }
       }
       return data;
@@ -3105,21 +3170,9 @@ public abstract class FileSystem extends Configured implements Closeable {
      */
     private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
       visitor.accept(rootData);
-      if (allData != null) {
-        for (Iterator<StatisticsData> iter = allData.iterator();
-            iter.hasNext(); ) {
-          StatisticsData data = iter.next();
-          visitor.accept(data);
-          if (data.owner.get() == null) {
-            /*
-             * If the thread that created this thread-local data no
-             * longer exists, remove the StatisticsData from our list
-             * and fold the values into rootData.
-             */
-            rootData.add(data);
-            iter.remove();
-          }
-        }
+      for (StatisticsDataReference ref: allData) {
+        StatisticsData data = ref.getData();
+        visitor.accept(data);
       }
       return visitor.aggregate();
     }
@@ -3226,7 +3279,7 @@ public abstract class FileSystem extends Configured implements Closeable {
     @Override
     public String toString() {
       return visitAll(new StatisticsAggregator<String>() {
-        private StatisticsData total = new StatisticsData(null);
+        private StatisticsData total = new StatisticsData();
 
         @Override
         public void accept(StatisticsData data) {
@@ -3259,7 +3312,7 @@ public abstract class FileSystem extends Configured implements Closeable {
      */
     public void reset() {
       visitAll(new StatisticsAggregator<Void>() {
-        private StatisticsData total = new StatisticsData(null);
+        private StatisticsData total = new StatisticsData();
 
         @Override
         public void accept(StatisticsData data) {
@@ -3281,6 +3334,11 @@ public abstract class FileSystem extends Configured implements Closeable {
     public String getScheme() {
       return scheme;
     }
+
+    @VisibleForTesting
+    synchronized int getAllThreadLocalDataSize() {
+      return allData.size();
+    }
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/296d1507/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 90337a6..3e33362 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -18,26 +18,34 @@
 
 package org.apache.hadoop.fs;
 
+import static org.apache.hadoop.fs.FileContextTestHelper.createFile;
+
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.google.common.base.Supplier;
 import com.google.common.util.concurrent.Uninterruptibles;
 
-import static org.apache.hadoop.fs.FileContextTestHelper.*;
-
 /**
  * <p>
  *   Base class to test {@link FileContext} Statistics.
  * </p>
  */
 public abstract class FCStatisticsBaseTest {
-  
   static protected int blockSize = 512;
   static protected int numBlocks = 1;
   
@@ -102,6 +110,48 @@ public abstract class FCStatisticsBaseTest {
     fc.delete(filePath, true);
   }
 
+  @Test(timeout=60000)
+  public void testStatisticsThreadLocalDataCleanUp() throws Exception {
+    final Statistics stats = new Statistics("test");
+    // create a small thread pool to test the statistics
+    final int size = 2;
+    ExecutorService es = Executors.newFixedThreadPool(size);
+    List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(size);
+    for (int i = 0; i < size; i++) {
+      tasks.add(new Callable<Boolean>() {
+        public Boolean call() {
+          // this populates the data set in statistics
+          stats.incrementReadOps(1);
+          return true;
+        }
+      });
+    }
+    // run the threads
+    es.invokeAll(tasks);
+    // assert that the data size is exactly the number of threads
+    final AtomicInteger allDataSize = new AtomicInteger(0);
+    allDataSize.set(stats.getAllThreadLocalDataSize());
+    Assert.assertEquals(size, allDataSize.get());
+    Assert.assertEquals(size, stats.getReadOps());
+    // force the GC to collect the threads by shutting down the thread pool
+    es.shutdownNow();
+    es.awaitTermination(1, TimeUnit.MINUTES);
+    es = null;
+    System.gc();
+
+    // wait for up to 10 seconds
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+          @Override
+          public Boolean get() {
+            int size = stats.getAllThreadLocalDataSize();
+            allDataSize.set(size);
+            return size == 0;
+          }
+        }, 1000, 10*1000);
+    Assert.assertEquals(0, allDataSize.get());
+    Assert.assertEquals(size, stats.getReadOps());
+  }
+
   /**
    * Bytes read may be different for different file systems. This method should
    * throw assertion error if bytes read are incorrect.


[22/48] hadoop git commit: HADOOP-12124. Add HTrace support for FsShell (cmccabe)

Posted by zj...@apache.org.
HADOOP-12124. Add HTrace support for FsShell (cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/349b9f98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/349b9f98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/349b9f98

Branch: refs/heads/YARN-2928
Commit: 349b9f984d0d3edf762491eb10cd54c1913ce2fd
Parents: 52d4f05
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:46:25 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt     |  2 ++
 .../src/main/java/org/apache/hadoop/fs/FsShell.java | 16 ++++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/349b9f98/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4d69a229..19ba620 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -671,6 +671,8 @@ Release 2.8.0 - UNRELEASED
     TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not
     installed. (wang)
 
+    HADOOP-12124. Add HTrace support for FsShell (cmccabe)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/349b9f98/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index db73f6d..24b6339 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -33,8 +33,13 @@ import org.apache.hadoop.fs.shell.Command;
 import org.apache.hadoop.fs.shell.CommandFactory;
 import org.apache.hadoop.fs.shell.FsCommand;
 import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.tracing.TraceUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.SamplerBuilder;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 /** Provide command line access to a FileSystem. */
 @InterfaceAudience.Private
@@ -47,6 +52,7 @@ public class FsShell extends Configured implements Tool {
   private FileSystem fs;
   private Trash trash;
   protected CommandFactory commandFactory;
+  private Sampler traceSampler;
 
   private final String usagePrefix =
     "Usage: hadoop fs [generic options]";
@@ -272,7 +278,8 @@ public class FsShell extends Configured implements Tool {
   public int run(String argv[]) throws Exception {
     // initialize FsShell
     init();
-
+    traceSampler = new SamplerBuilder(TraceUtils.
+        wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
     int exitCode = -1;
     if (argv.length < 1) {
       printUsage(System.err);
@@ -284,7 +291,12 @@ public class FsShell extends Configured implements Tool {
         if (instance == null) {
           throw new UnknownCommandException();
         }
-        exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));
+        TraceScope scope = Trace.startSpan(instance.getCommandName(), traceSampler);
+        try {
+          exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));
+        } finally {
+          scope.close();
+        }
       } catch (IllegalArgumentException e) {
         displayError(cmd, e.getLocalizedMessage());
         if (instance != null) {


[26/48] hadoop git commit: YARN-3823. Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray Chiang.

Posted by zj...@apache.org.
YARN-3823. Fix mismatch in default values for
yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray
Chiang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09cd3dfc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09cd3dfc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09cd3dfc

Branch: refs/heads/YARN-2928
Commit: 09cd3dfcdc3bdd0dd7f3f2223c30768a3f273ed9
Parents: 2176540
Author: Devaraj K <de...@apache.org>
Authored: Wed Jul 1 11:11:26 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:58 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                                   | 3 +++
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml        | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/09cd3dfc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5269bec..abba1b8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables.
     (Zhihai Xu via gera)
 
+    YARN-3823. Fix mismatch in default values for 
+    yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09cd3dfc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d34cf82..62527b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -276,7 +276,7 @@
     in terms of virtual CPU cores. Requests higher than this will throw a
     InvalidResourceRequestException.</description>
     <name>yarn.scheduler.maximum-allocation-vcores</name>
-    <value>32</value>
+    <value>4</value>
   </property>
 
   <property>


[16/48] hadoop git commit: YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables. (Zhihai Xu via gera)

Posted by zj...@apache.org.
YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables. (Zhihai Xu via gera)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcac9c96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcac9c96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcac9c96

Branch: refs/heads/YARN-2928
Commit: bcac9c96b4e228239d364f583849cf6a788ffc6a
Parents: 9465dc4
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Jun 30 14:44:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:56 2015 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/util/Shell.java |  6 +-
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../java/org/apache/hadoop/yarn/util/Apps.java  | 19 ++++--
 .../org/apache/hadoop/yarn/util/TestApps.java   | 61 ++++++++++++++++++++
 4 files changed, 81 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 45c1588..ed83e8d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -221,10 +221,12 @@ abstract public class Shell {
       new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
   }
 
+  public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
   /** Return a regular expression string that match environment variables */
   public static String getEnvironmentVariableRegex() {
-    return (WINDOWS) ? "%([A-Za-z_][A-Za-z0-9_]*?)%" :
-      "\\$([A-Za-z_][A-Za-z0-9_]*)";
+    return (WINDOWS)
+        ? "%(" + ENV_NAME_REGEX + "?)%"
+        : "\\$(" + ENV_NAME_REGEX + ")";
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d8cfa4b..896f54e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -677,6 +677,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3770. SerializedException should also handle java.lang.Error on
     de-serialization. (Lavkesh Lahngir via jianhe)
 
+    YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables.
+    (Zhihai Xu via gera)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index cf3940f..9235e7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -44,6 +44,14 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 public class Apps {
   public static final String APP = "application";
   public static final String ID = "ID";
+  private static final Pattern VAR_SUBBER =
+      Pattern.compile(Shell.getEnvironmentVariableRegex());
+  private static final Pattern VARVAL_SPLITTER = Pattern.compile(
+        "(?<=^|,)"                            // preceded by ',' or line begin
+      + '(' + Shell.ENV_NAME_REGEX + ')'      // var group
+      + '='
+      + "([^,]*)"                             // val group
+      );
 
   public static ApplicationId toAppID(String aid) {
     Iterator<String> it = _split(aid).iterator();
@@ -73,11 +81,10 @@ public class Apps {
   public static void setEnvFromInputString(Map<String, String> env,
       String envString,  String classPathSeparator) {
     if (envString != null && envString.length() > 0) {
-      String childEnvs[] = envString.split(",");
-      Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
-      for (String cEnv : childEnvs) {
-        String[] parts = cEnv.split("="); // split on '='
-        Matcher m = p.matcher(parts[1]);
+      Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+      while (varValMatcher.find()) {
+        String envVar = varValMatcher.group(1);
+        Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
         StringBuffer sb = new StringBuffer();
         while (m.find()) {
           String var = m.group(1);
@@ -93,7 +100,7 @@ public class Apps {
           m.appendReplacement(sb, Matcher.quoteReplacement(replace));
         }
         m.appendTail(sb);
-        addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+        addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcac9c96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
new file mode 100644
index 0000000..66d2d23
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.util;
+
+import org.apache.hadoop.util.Shell;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestApps {
+  @Test
+  public void testSetEnvFromInputString() {
+    Map<String, String> environment = new HashMap<String, String>();
+    environment.put("JAVA_HOME", "/path/jdk");
+    String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
+        + ",g_nix=$JAVA_HOME";
+    Apps.setEnvFromInputString(environment, goodEnv, File.pathSeparator);
+    assertEquals("1", environment.get("a1"));
+    assertEquals("2", environment.get("b_2"));
+    assertEquals("3", environment.get("_c"));
+    assertEquals("4", environment.get("d"));
+    assertEquals("", environment.get("e"));
+    if (Shell.WINDOWS) {
+      assertEquals("$JAVA_HOME", environment.get("g_nix"));
+      assertEquals("/path/jdk", environment.get("f_win"));
+    } else {
+      assertEquals("/path/jdk", environment.get("g_nix"));
+      assertEquals("%JAVA_HOME%", environment.get("f_win"));
+    }
+    String badEnv = "1,,2=a=b,3=a=,4==,5==a,==,c-3=3,=";
+    environment.clear();
+    Apps.setEnvFromInputString(environment, badEnv, File.pathSeparator);
+    assertEquals(environment.size(), 0);
+
+    // Test "=" in the value part
+    environment.clear();
+    Apps.setEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2",
+        File.pathSeparator);
+    assertEquals("=", environment.get("e1"));
+    assertEquals("a1=a2", environment.get("e2"));
+  }
+}


[35/48] hadoop git commit: YARN-3793. Several NPEs when deleting local files on NM recovery. Contributed by Varun Saxena

Posted by zj...@apache.org.
YARN-3793. Several NPEs when deleting local files on NM recovery. Contributed by Varun Saxena


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e556ee03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e556ee03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e556ee03

Branch: refs/heads/YARN-2928
Commit: e556ee030ff3610c8c407caa18eddd2a6c443d9d
Parents: d7e6226
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 21:13:32 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../logaggregation/AppLogAggregatorImpl.java    |  6 ++--
 .../TestLogAggregationService.java              | 36 ++++++++++++++++++++
 3 files changed, 42 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fff46b8..c4c7fa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -701,6 +701,9 @@ Release 2.7.2 - UNRELEASED
 
   BUG FIXES
 
+    YARN-3793. Several NPEs when deleting local files on NM recovery (Varun
+    Saxena via jlowe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 4b95a03..654eb0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -276,10 +276,10 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
             aggregator.doContainerLogAggregation(writer, appFinished);
         if (uploadedFilePathsInThisCycle.size() > 0) {
           uploadedLogsInThisCycle = true;
+          this.delService.delete(this.userUgi.getShortUserName(), null,
+              uploadedFilePathsInThisCycle
+                  .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
         }
-        this.delService.delete(this.userUgi.getShortUserName(), null,
-          uploadedFilePathsInThisCycle
-            .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
 
         // This container is finished, and all its logs have been uploaded,
         // remove it from containerLogAggregators.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index fd97cef..6a3d270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -31,6 +31,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -280,6 +281,41 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
     verifyLocalFileDeletion(logAggregationService);
   }
 
+  /* Test to verify fix for YARN-3793 */
+  @Test
+  public void testNoLogsUploadedOnAppFinish() throws Exception {
+    this.delSrvc = new DeletionService(createContainerExecutor());
+    delSrvc = spy(delSrvc);
+    this.delSrvc.init(conf);
+    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+        this.remoteRootLogDir.getAbsolutePath());
+
+    LogAggregationService logAggregationService = new LogAggregationService(
+        dispatcher, this.context, this.delSrvc, super.dirsHandler);
+    logAggregationService.init(this.conf);
+    logAggregationService.start();
+
+    ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
+    File appLogDir = new File(localLogDir, ConverterUtils.toString(app));
+    appLogDir.mkdir();
+    LogAggregationContext context =
+        LogAggregationContext.newInstance("HOST*", "sys*");
+    logAggregationService.handle(new LogHandlerAppStartedEvent(app, this.user,
+        null, ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls, context));
+
+    ApplicationAttemptId appAttemptId =
+        BuilderUtils.newApplicationAttemptId(app, 1);
+    ContainerId cont = BuilderUtils.newContainerId(appAttemptId, 1);
+    writeContainerLogs(appLogDir, cont, new String[] { "stdout",
+        "stderr", "syslog" });
+    logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont, 0));
+    logAggregationService.handle(new LogHandlerAppFinishedEvent(app));
+    logAggregationService.stop();
+    delSrvc.stop();
+    // Aggregated logs should not be deleted if not uploaded.
+    verify(delSrvc, times(0)).delete(user, null);
+  }
 
   @Test
   public void testNoContainerOnNode() throws Exception {


[44/48] hadoop git commit: HDFS-8686. WebHdfsFileSystem#getXAttr(Path p, final String name) doesn't work if namespace is in capitals (Contributed by kanaka kumar avvaru)

Posted by zj...@apache.org.
HDFS-8686. WebHdfsFileSystem#getXAttr(Path p, final String name) doesn't work if namespace is in capitals (Contributed by kanaka kumar avvaru)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68375526
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68375526
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68375526

Branch: refs/heads/YARN-2928
Commit: 68375526908332b9a5ec12228937b22dc18e82a2
Parents: 7182a03
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jul 6 16:09:24 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hdfs/web/JsonUtilClient.java    | 14 ++++++++++++++
 .../org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java |  2 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt           |  3 +++
 .../hadoop/hdfs/server/namenode/FSXAttrBaseTest.java  |  5 ++++-
 .../TestOfflineImageViewerForXAttr.java               |  3 +++
 5 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index e025e31..713836c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -413,6 +413,20 @@ class JsonUtilClient {
     return null;
   }
 
+  /** Expecting only single XAttr in the map. return its value */
+  static byte[] getXAttr(final Map<?, ?> json) throws IOException {
+    if (json == null) {
+      return null;
+    }
+
+    Map<String, byte[]> xAttrs = toXAttrs(json);
+    if (xAttrs != null && !xAttrs.values().isEmpty()) {
+      return xAttrs.values().iterator().next();
+    }
+
+    return null;
+  }
+
   static Map<String, byte[]> toXAttrs(final Map<?, ?> json)
       throws IOException {
     if (json == null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 2650dca..b661d07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -963,7 +963,7 @@ public class WebHdfsFileSystem extends FileSystem
         new XAttrEncodingParam(XAttrCodec.HEX)) {
       @Override
       byte[] decodeResponse(Map<?, ?> json) throws IOException {
-        return JsonUtilClient.getXAttr(json, name);
+        return JsonUtilClient.getXAttr(json);
       }
     }.run();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4f184fb..9edc2af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1005,6 +1005,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8577. Avoid retrying to recover lease on a file which does not exist
     (J.Andreina via vinayakumarb)
 
+    HDFS-8686. WebHdfsFileSystem#getXAttr(Path p, final String name) doesn't
+    work if namespace is in capitals (kanaka kumar avvaru via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
index e21e34c..eb9053c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
@@ -395,7 +395,10 @@ public class FSXAttrBaseTest {
     FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
     fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE));
     fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE));
-    
+
+    final byte[] theValue = fs.getXAttr(path, "USER.a2");
+    Assert.assertArrayEquals(value2, theValue);
+
     /* An XAttr that was requested does not exist. */
     try {
       final byte[] value = fs.getXAttr(path, name3);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68375526/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
index 3f23f64..6c82101 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
@@ -231,6 +231,9 @@ public class TestOfflineImageViewerForXAttr {
           "user.attr1"));
       assertEquals("value1", value);
 
+      value = new String(webhdfs.getXAttr(new Path("/dir1"), "USER.attr1"));
+      assertEquals("value1", value);
+
       Map<String, byte[]> contentMap = webhdfs.getXAttrs(new Path("/dir1"),
           names);
 


[47/48] hadoop git commit: YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner after create them. Contributed by zhihai xu

Posted by zj...@apache.org.
YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner
after create them. Contributed by zhihai xu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccfc7874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccfc7874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccfc7874

Branch: refs/heads/YARN-2928
Commit: ccfc78748569b49ffa34dcb64032222174489058
Parents: 06daeae
Author: Xuan <xg...@apache.org>
Authored: Sat Jul 4 21:51:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:02 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../logaggregation/AggregatedLogFormat.java     | 83 +++++++++++---------
 2 files changed, 49 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccfc7874/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9780b4f..91358c1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -692,6 +692,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
     properly in log. (Bibin A Chundatt via devaraj)
 
+    YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner
+    after create them. (zhihai xu via xgong)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccfc7874/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index debe770..c9453b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -489,18 +489,23 @@ public class AggregatedLogFormat {
      * @throws IOException
      */
     public String getApplicationOwner() throws IOException {
-      TFile.Reader.Scanner ownerScanner = reader.createScanner();
-      LogKey key = new LogKey();
-      while (!ownerScanner.atEnd()) {
-        TFile.Reader.Scanner.Entry entry = ownerScanner.entry();
-        key.readFields(entry.getKeyStream());
-        if (key.toString().equals(APPLICATION_OWNER_KEY.toString())) {
-          DataInputStream valueStream = entry.getValueStream();
-          return valueStream.readUTF();
+      TFile.Reader.Scanner ownerScanner = null;
+      try {
+        ownerScanner = reader.createScanner();
+        LogKey key = new LogKey();
+        while (!ownerScanner.atEnd()) {
+          TFile.Reader.Scanner.Entry entry = ownerScanner.entry();
+          key.readFields(entry.getKeyStream());
+          if (key.toString().equals(APPLICATION_OWNER_KEY.toString())) {
+            DataInputStream valueStream = entry.getValueStream();
+            return valueStream.readUTF();
+          }
+          ownerScanner.advance();
         }
-        ownerScanner.advance();
+        return null;
+      } finally {
+        IOUtils.cleanup(LOG, ownerScanner);
       }
-      return null;
     }
 
     /**
@@ -513,38 +518,42 @@ public class AggregatedLogFormat {
     public Map<ApplicationAccessType, String> getApplicationAcls()
         throws IOException {
       // TODO Seek directly to the key once a comparator is specified.
-      TFile.Reader.Scanner aclScanner = reader.createScanner();
-      LogKey key = new LogKey();
-      Map<ApplicationAccessType, String> acls =
-          new HashMap<ApplicationAccessType, String>();
-      while (!aclScanner.atEnd()) {
-        TFile.Reader.Scanner.Entry entry = aclScanner.entry();
-        key.readFields(entry.getKeyStream());
-        if (key.toString().equals(APPLICATION_ACL_KEY.toString())) {
-          DataInputStream valueStream = entry.getValueStream();
-          while (true) {
-            String appAccessOp = null;
-            String aclString = null;
-            try {
-              appAccessOp = valueStream.readUTF();
-            } catch (EOFException e) {
-              // Valid end of stream.
-              break;
-            }
-            try {
-              aclString = valueStream.readUTF();
-            } catch (EOFException e) {
-              throw new YarnRuntimeException("Error reading ACLs", e);
+      TFile.Reader.Scanner aclScanner = null;
+      try {
+        aclScanner = reader.createScanner();
+        LogKey key = new LogKey();
+        Map<ApplicationAccessType, String> acls =
+            new HashMap<ApplicationAccessType, String>();
+        while (!aclScanner.atEnd()) {
+          TFile.Reader.Scanner.Entry entry = aclScanner.entry();
+          key.readFields(entry.getKeyStream());
+          if (key.toString().equals(APPLICATION_ACL_KEY.toString())) {
+            DataInputStream valueStream = entry.getValueStream();
+            while (true) {
+              String appAccessOp = null;
+              String aclString = null;
+              try {
+                appAccessOp = valueStream.readUTF();
+              } catch (EOFException e) {
+                // Valid end of stream.
+                break;
+              }
+              try {
+                aclString = valueStream.readUTF();
+              } catch (EOFException e) {
+                throw new YarnRuntimeException("Error reading ACLs", e);
+              }
+              acls.put(ApplicationAccessType.valueOf(appAccessOp), aclString);
             }
-            acls.put(ApplicationAccessType.valueOf(appAccessOp), aclString);
           }
-
+          aclScanner.advance();
         }
-        aclScanner.advance();
+        return acls;
+      } finally {
+        IOUtils.cleanup(LOG, aclScanner);
       }
-      return acls;
     }
-    
+
     /**
      * Read the next key and return the value-stream.
      * 


[04/48] hadoop git commit: HADOOP-12112. Make hadoop-common-project Native code -Wall-clean (alanburlison via cmccabe)

Posted by zj...@apache.org.
HADOOP-12112. Make hadoop-common-project Native code -Wall-clean (alanburlison via cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38d1678e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38d1678e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38d1678e

Branch: refs/heads/YARN-2928
Commit: 38d1678e94d5babba13693a8350140f91b02b567
Parents: 96d07d2
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 29 12:34:47 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:30:01 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt              | 3 +++
 .../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 1 +
 .../native/src/org/apache/hadoop/net/unix/DomainSocket.c     | 8 ++++----
 .../src/main/native/src/org/apache/hadoop/util/NativeCrc32.c | 4 ++--
 4 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e8e85a0..9008ead 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12036. Consolidate all of the cmake extensions in one directory
     (alanburlison via cmccabe)
 
+    HADOOP-12112. Make hadoop-common-project Native code -Wall-clean
+    (alanburlison via cmccabe)
+
   BUG FIXES
 
     HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index 071d830..bc78ab2 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -592,6 +592,7 @@ done:
 #else
   THROW(env, "java/io/IOException",
     "The function Windows.createFileWithMode0() is not supported on this platform");
+  return NULL;
 #endif
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
index 55ab0c3..a3f27ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
@@ -112,9 +112,9 @@ static jthrowable newSocketException(JNIEnv *env, int errnum,
  * to the heap if necessary.
  */
 struct flexibleBuffer {
-  int8_t *curBuf;
-  int8_t *allocBuf;
-  int8_t stackBuf[8196];
+  jbyte *curBuf;
+  jbyte *allocBuf;
+  jbyte stackBuf[8196];
 };
 
 static jthrowable flexBufInit(JNIEnv *env, struct flexibleBuffer *flexBuf, jint length)
@@ -637,7 +637,7 @@ JNIEnv *env, jclass clazz, jint fd)
  * @return               NULL on success; or the unraised exception representing
  *                       the problem.
  */
-static jthrowable write_fully(JNIEnv *env, int fd, int8_t *buf, int amt)
+static jthrowable write_fully(JNIEnv *env, int fd, jbyte *buf, int amt)
 {
   int err, res;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38d1678e/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
index 74e09e6..3fceb02 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
@@ -168,7 +168,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChun
   // Setup complete. Actually verify checksums.
   ret = bulk_crc(data, data_len, sums, crc_type,
                             bytes_per_checksum, verify ? &error_data : NULL);
-  if (likely(verify && ret == CHECKSUMS_VALID || !verify && ret == 0)) {
+  if (likely((verify && ret == CHECKSUMS_VALID) || (!verify && ret == 0))) {
     return;
   } else if (unlikely(verify && ret == INVALID_CHECKSUM_DETECTED)) {
     long pos = base_pos + (error_data.bad_data - data);
@@ -261,7 +261,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChun
         env, error_data.got_crc, error_data.expected_crc,
         j_filename, pos);
       return;
-    } else if (unlikely(verify && ret != CHECKSUMS_VALID || !verify && ret != 0)) {
+    } else if (unlikely((verify && ret != CHECKSUMS_VALID) || (!verify && ret != 0))) {
       THROW(env, "java/lang/AssertionError",
         "Bad response code from native bulk_crc");
       return;


[38/48] hadoop git commit: HADOOP-12171. Shorten overly-long htrace span names for server (cmccabe)

Posted by zj...@apache.org.
HADOOP-12171. Shorten overly-long htrace span names for server (cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f530f52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f530f52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f530f52

Branch: refs/heads/YARN-2928
Commit: 6f530f52f0a4faf7887d36e2f1a7c70b1fb4e1fc
Parents: d5192ca
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Jul 1 17:57:11 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  2 ++
 .../org/apache/hadoop/ipc/RpcClientUtil.java    | 24 ++++++++++++++++++++
 .../main/java/org/apache/hadoop/ipc/Server.java |  4 +++-
 .../org/apache/hadoop/tracing/TestTracing.java  | 18 +++++++--------
 4 files changed, 38 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 39e2e5e..24431ba 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -673,6 +673,8 @@ Release 2.8.0 - UNRELEASED
 
     HADOOP-12124. Add HTrace support for FsShell (cmccabe)
 
+    HADOOP-12171. Shorten overly-long htrace span names for server (cmccabe)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
index d9bd71b..da1e699 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
@@ -210,4 +210,28 @@ public class RpcClientUtil {
     }
     return clazz.getSimpleName() + "#" + method.getName();
   }
+
+  /**
+   * Convert an RPC class method to a string.
+   * The format we want is
+   * 'SecondOutermostClassShortName#OutermostClassShortName'.
+   *
+   * For example, if the full class name is:
+   *   org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations
+   *
+   * the format we want is:
+   *   ClientProtocol#getBlockLocations
+   */
+  public static String toTraceName(String fullName) {
+    int lastPeriod = fullName.lastIndexOf('.');
+    if (lastPeriod < 0) {
+      return fullName;
+    }
+    int secondLastPeriod = fullName.lastIndexOf('.', lastPeriod - 1);
+    if (secondLastPeriod < 0) {
+      return fullName;
+    }
+    return fullName.substring(secondLastPeriod + 1, lastPeriod) + "#" +
+        fullName.substring(lastPeriod + 1);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 98fffc0..4026fe0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -1963,7 +1963,9 @@ public abstract class Server {
         // If the incoming RPC included tracing info, always continue the trace
         TraceInfo parentSpan = new TraceInfo(header.getTraceInfo().getTraceId(),
                                              header.getTraceInfo().getParentId());
-        traceSpan = Trace.startSpan(rpcRequest.toString(), parentSpan).detach();
+        traceSpan = Trace.startSpan(
+            RpcClientUtil.toTraceName(rpcRequest.toString()),
+            parentSpan).detach();
       }
 
       Call call = new Call(header.getCallId(), header.getRetryCount(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f530f52/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 58b3659..c3d2c73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -67,18 +67,18 @@ public class TestTracing {
 
     String[] expectedSpanNames = {
       "testWriteTraceHooks",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.create",
+      "ClientProtocol#create",
       "ClientNamenodeProtocol#create",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.fsync",
+      "ClientProtocol#fsync",
       "ClientNamenodeProtocol#fsync",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
+      "ClientProtocol#complete",
       "ClientNamenodeProtocol#complete",
       "newStreamForCreate",
       "DFSOutputStream#write",
       "DFSOutputStream#close",
       "dataStreamer",
       "OpWriteBlockProto",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.addBlock",
+      "ClientProtocol#addBlock",
       "ClientNamenodeProtocol#addBlock"
     };
     SetSpanReceiver.assertSpanNamesFound(expectedSpanNames);
@@ -95,11 +95,11 @@ public class TestTracing {
     // and children of them are exception.
     String[] spansInTopTrace = {
       "testWriteTraceHooks",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.create",
+      "ClientProtocol#create",
       "ClientNamenodeProtocol#create",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.fsync",
+      "ClientProtocol#fsync",
       "ClientNamenodeProtocol#fsync",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
+      "ClientProtocol#complete",
       "ClientNamenodeProtocol#complete",
       "newStreamForCreate",
       "DFSOutputStream#write",
@@ -113,7 +113,7 @@ public class TestTracing {
 
     // test for timeline annotation added by HADOOP-11242
     Assert.assertEquals("called",
-        map.get("org.apache.hadoop.hdfs.protocol.ClientProtocol.create")
+        map.get("ClientProtocol#create")
            .get(0).getTimelineAnnotations()
            .get(0).getMessage());
 
@@ -131,7 +131,7 @@ public class TestTracing {
 
     String[] expectedSpanNames = {
       "testReadTraceHooks",
-      "org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations",
+      "ClientProtocol#getBlockLocations",
       "ClientNamenodeProtocol#getBlockLocations",
       "OpReadBlockProto"
     };


[21/48] hadoop git commit: HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil and fix for HA namespaces (rchiang via rkanter)

Posted by zj...@apache.org.
HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil and fix for HA namespaces (rchiang via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c906166f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c906166f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c906166f

Branch: refs/heads/YARN-2928
Commit: c906166f7ebaa851283f391b62be50976c71a2bd
Parents: 349b9f9
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 30 16:42:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../java/org/apache/hadoop/fs/FileUtil.java     |  42 +++++++
 .../java/org/apache/hadoop/fs/TestFileUtil.java | 112 ++++++++++++++++++-
 .../java/org/apache/hadoop/tools/DistCp.java    |   3 +-
 .../apache/hadoop/tools/util/DistCpUtils.java   |  39 -------
 5 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 19ba620..4e5f6bb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -910,6 +910,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-10798. globStatus() should always return a sorted list of files
     (cmccabe)
 
+    HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil
+    and fix for HA namespaces (rchiang via rkanter)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 9b9e213..8abb4eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -19,6 +19,9 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
@@ -1329,4 +1332,43 @@ public class FileUtil {
                         unexpandedWildcardClasspath.toString()};
     return jarCp;
   }
+
+  public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
+    if (srcFs==null || destFs==null) {
+      return false;
+    }
+    URI srcUri = srcFs.getUri();
+    URI dstUri = destFs.getUri();
+    if (srcUri.getScheme()==null) {
+      return false;
+    }
+    if (!srcUri.getScheme().equals(dstUri.getScheme())) {
+      return false;
+    }
+    String srcHost = srcUri.getHost();
+    String dstHost = dstUri.getHost();
+    if ((srcHost!=null) && (dstHost!=null)) {
+      if (srcHost.equals(dstHost)) {
+        return srcUri.getPort()==dstUri.getPort();
+      }
+      try {
+        srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
+        dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
+      } catch (UnknownHostException ue) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Could not compare file-systems. Unknown host: ", ue);
+        }
+        return false;
+      }
+      if (!srcHost.equals(dstHost)) {
+        return false;
+      }
+    } else if (srcHost==null && dstHost!=null) {
+      return false;
+    } else if (srcHost!=null) {
+      return false;
+    }
+    // check for ports
+    return srcUri.getPort()==dstUri.getPort();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 41794b8..3418ade 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -17,7 +17,8 @@
  */
 package org.apache.hadoop.fs;
 
-import org.junit.Before;
+import org.junit.*;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -25,8 +26,11 @@ import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.net.InetAddress;
 import java.net.URI;
 import java.io.PrintWriter;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -44,10 +48,12 @@ import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.tools.tar.TarEntry;
 import org.apache.tools.tar.TarOutputStream;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
+
+import javax.print.attribute.URISyntax;
+
 import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TestFileUtil {
   private static final Log LOG = LogFactory.getLog(TestFileUtil.class);
@@ -64,6 +70,25 @@ public class TestFileUtil {
   private final File dir2 = new File(del, DIR + "2");
   private final File partitioned = new File(TEST_DIR, "partitioned");
 
+  private InetAddress inet1;
+  private InetAddress inet2;
+  private InetAddress inet3;
+  private InetAddress inet4;
+  private InetAddress inet5;
+  private InetAddress inet6;
+  private URI uri1;
+  private URI uri2;
+  private URI uri3;
+  private URI uri4;
+  private URI uri5;
+  private URI uri6;
+  private FileSystem fs1;
+  private FileSystem fs2;
+  private FileSystem fs3;
+  private FileSystem fs4;
+  private FileSystem fs5;
+  private FileSystem fs6;
+
   /**
    * Creates multiple directories for testing.
    * 
@@ -80,6 +105,7 @@ public class TestFileUtil {
    *   file: part-r-00000, contents: "foo"
    *   file: part-r-00001, contents: "bar"
    */
+  @Ignore
   private void setupDirs() throws IOException {
     Assert.assertFalse(del.exists());
     Assert.assertFalse(tmp.exists());
@@ -1096,4 +1122,82 @@ public class TestFileUtil {
       }
     }
   }
+
+  @Ignore
+  public void setupCompareFs() {
+    // Set up Strings
+    String host1 = "1.2.3.4";
+    String host2 = "2.3.4.5";
+    int port1 = 7000;
+    int port2 = 7001;
+    String uris1 = "hdfs://" + host1 + ":" + Integer.toString(port1) + "/tmp/foo";
+    String uris2 = "hdfs://" + host1 + ":" + Integer.toString(port2) + "/tmp/foo";
+    String uris3 = "hdfs://" + host2 + ":" + Integer.toString(port2) + "/tmp/foo";
+    String uris4 = "hdfs://" + host2 + ":" + Integer.toString(port2) + "/tmp/foo";
+    String uris5 = "file:///" + host1 + ":" + Integer.toString(port1) + "/tmp/foo";
+    String uris6 = "hdfs:///" + host1 + "/tmp/foo";
+    // Set up URI objects
+    try {
+      uri1 = new URI(uris1);
+      uri2 = new URI(uris2);
+      uri3 = new URI(uris3);
+      uri4 = new URI(uris4);
+      uri5 = new URI(uris5);
+      uri6 = new URI(uris6);
+    } catch (URISyntaxException use) {
+    }
+    // Set up InetAddress
+    inet1 = mock(InetAddress.class);
+    when(inet1.getCanonicalHostName()).thenReturn(host1);
+    inet2 = mock(InetAddress.class);
+    when(inet2.getCanonicalHostName()).thenReturn(host1);
+    inet3 = mock(InetAddress.class);
+    when(inet3.getCanonicalHostName()).thenReturn(host2);
+    inet4 = mock(InetAddress.class);
+    when(inet4.getCanonicalHostName()).thenReturn(host2);
+    inet5 = mock(InetAddress.class);
+    when(inet5.getCanonicalHostName()).thenReturn(host1);
+    inet6 = mock(InetAddress.class);
+    when(inet6.getCanonicalHostName()).thenReturn(host1);
+
+    // Link of InetAddress to corresponding URI
+    try {
+      when(InetAddress.getByName(uris1)).thenReturn(inet1);
+      when(InetAddress.getByName(uris2)).thenReturn(inet2);
+      when(InetAddress.getByName(uris3)).thenReturn(inet3);
+      when(InetAddress.getByName(uris4)).thenReturn(inet4);
+      when(InetAddress.getByName(uris5)).thenReturn(inet5);
+    } catch (UnknownHostException ue) {
+    }
+
+    fs1 = mock(FileSystem.class);
+    when(fs1.getUri()).thenReturn(uri1);
+    fs2 = mock(FileSystem.class);
+    when(fs2.getUri()).thenReturn(uri2);
+    fs3 = mock(FileSystem.class);
+    when(fs3.getUri()).thenReturn(uri3);
+    fs4 = mock(FileSystem.class);
+    when(fs4.getUri()).thenReturn(uri4);
+    fs5 = mock(FileSystem.class);
+    when(fs5.getUri()).thenReturn(uri5);
+    fs6 = mock(FileSystem.class);
+    when(fs6.getUri()).thenReturn(uri6);
+  }
+
+  @Test
+  public void testCompareFsNull() throws Exception {
+    setupCompareFs();
+    assertEquals(FileUtil.compareFs(null,fs1),false);
+    assertEquals(FileUtil.compareFs(fs1,null),false);
+  }
+
+  @Test
+  public void testCompareFsDirectories() throws Exception {
+    setupCompareFs();
+    assertEquals(FileUtil.compareFs(fs1,fs1),true);
+    assertEquals(FileUtil.compareFs(fs1,fs2),false);
+    assertEquals(FileUtil.compareFs(fs1,fs5),false);
+    assertEquals(FileUtil.compareFs(fs3,fs4),true);
+    assertEquals(FileUtil.compareFs(fs1,fs6),false);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 6921a1e..2385df3 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Cluster;
@@ -347,7 +348,7 @@ public class DistCp extends Configured implements Tool {
       workDir = new Path(workDir, WIP_PREFIX + targetPath.getName()
                                 + rand.nextInt());
       FileSystem workFS = workDir.getFileSystem(configuration);
-      if (!DistCpUtils.compareFs(targetFS, workFS)) {
+      if (!FileUtil.compareFs(targetFS, workFS)) {
         throw new IllegalArgumentException("Work path " + workDir +
             " and target path " + targetPath + " are in different file system");
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c906166f/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index 5ac5bf1..70d947e 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -468,43 +468,4 @@ public class DistCpUtils {
     return (sourceChecksum == null || targetChecksum == null ||
             sourceChecksum.equals(targetChecksum));
   }
-
-  /* see if two file systems are the same or not
-   *
-   */
-  public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
-    URI srcUri = srcFs.getUri();
-    URI dstUri = destFs.getUri();
-    if (srcUri.getScheme() == null) {
-      return false;
-    }
-    if (!srcUri.getScheme().equals(dstUri.getScheme())) {
-      return false;
-    }
-    String srcHost = srcUri.getHost();
-    String dstHost = dstUri.getHost();
-    if ((srcHost != null) && (dstHost != null)) {
-      try {
-        srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
-        dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
-      } catch(UnknownHostException ue) {
-        if (LOG.isDebugEnabled())
-          LOG.debug("Could not compare file-systems. Unknown host: ", ue);
-        return false;
-      }
-      if (!srcHost.equals(dstHost)) {
-        return false;
-      }
-    }
-    else if (srcHost == null && dstHost != null) {
-      return false;
-    }
-    else if (srcHost != null) {
-      return false;
-    }
-
-    //check for ports
-
-    return srcUri.getPort() == dstUri.getPort();
-  }
 }


[25/48] hadoop git commit: HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)

Posted by zj...@apache.org.
HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52d4f05a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52d4f05a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52d4f05a

Branch: refs/heads/YARN-2928
Commit: 52d4f05a83024cbe78eff6c58048c9595261a27f
Parents: 0e713f2
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:40:21 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../main/java/org/apache/hadoop/fs/Globber.java | 11 ++++++++-
 .../org/apache/hadoop/fs/TestGlobPaths.java     | 25 ++++++++++++++++++++
 3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2aab8b4..4d69a229 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
     (J.Andreina via vinayakumarb)
 
+    HADOOP-10798. globStatus() should always return a sorted list of files
+    (cmccabe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index 8a8137a..9cb810f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.LogFactory;
@@ -285,6 +286,14 @@ class Globber {
         (flattenedPatterns.size() <= 1)) {
       return null;
     }
-    return results.toArray(new FileStatus[0]);
+    /*
+     * In general, the results list will already be sorted, since listStatus
+     * returns results in sorted order for many Hadoop filesystems.  However,
+     * not all Hadoop filesystems have this property.  So we sort here in order
+     * to get consistent results.  See HADOOP-10798 for details.
+     */
+    FileStatus ret[] = results.toArray(new FileStatus[0]);
+    Arrays.sort(ret);
+    return ret;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
index 50e2e5b..afd8fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
@@ -21,9 +21,11 @@ import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
 import java.util.UUID;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.Ordering;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -1284,4 +1286,27 @@ public class TestGlobPaths {
   public void testNonTerminalGlobsOnFC() throws Exception {
     testOnFileContext(new TestNonTerminalGlobs(true));
   }
+
+  @Test
+  public void testLocalFilesystem() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem fs = FileSystem.getLocal(conf);
+    String localTmp = System.getProperty("java.io.tmpdir");
+    Path base = new Path(new Path(localTmp), UUID.randomUUID().toString());
+    Assert.assertTrue(fs.mkdirs(base));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "e")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "c")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "a")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "d")));
+    Assert.assertTrue(fs.mkdirs(new Path(base, "b")));
+    fs.deleteOnExit(base);
+    FileStatus[] status = fs.globStatus(new Path(base, "*"));
+    ArrayList list = new ArrayList();
+    for (FileStatus f: status) {
+        list.add(f.getPath().toString());
+    }
+    boolean sorted = Ordering.natural().isOrdered(list);
+    Assert.assertTrue(sorted);
+  }
 }
+


[42/48] hadoop git commit: HADOOP-12172. FsShell mkdir -p makes an unnecessary check for the existence of the parent. Contributed by Chris Nauroth.

Posted by zj...@apache.org.
HADOOP-12172. FsShell mkdir -p makes an unnecessary check for the existence of the parent. Contributed by Chris Nauroth.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8a65745
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8a65745
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8a65745

Branch: refs/heads/YARN-2928
Commit: a8a65745c40bc662065e140664062ca7dad3b9f2
Parents: 6f530f5
Author: cnauroth <cn...@apache.org>
Authored: Wed Jul 1 19:47:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                   | 3 +++
 .../src/main/java/org/apache/hadoop/fs/shell/Mkdir.java           | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8a65745/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 24431ba..312a996 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -704,6 +704,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12112. Make hadoop-common-project Native code -Wall-clean
     (alanburlison via cmccabe)
 
+    HADOOP-12172. FsShell mkdir -p makes an unnecessary check for the existence
+    of the parent. (cnauroth)
+
   BUG FIXES
 
     HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8a65745/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
index 74bad62..9f39da2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
@@ -70,7 +70,8 @@ class Mkdir extends FsCommand {
   protected void processNonexistentPath(PathData item) throws IOException {
     // check if parent exists. this is complicated because getParent(a/b/c/) returns a/b/c, but
     // we want a/b
-    if (!item.fs.exists(new Path(item.path.toString()).getParent()) && !createParents) {
+    if (!createParents &&
+        !item.fs.exists(new Path(item.path.toString()).getParent())) {
       throw new PathNotFoundException(item.toString());
     }
     if (!item.fs.mkdirs(item.path)) {


[12/48] hadoop git commit: HDFS-8627. NPE thrown if unable to fetch token from Namenode (Contributed by J.Andreina)

Posted by zj...@apache.org.
HDFS-8627. NPE thrown if unable to fetch token from Namenode (Contributed by J.Andreina)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9634540
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9634540
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9634540

Branch: refs/heads/YARN-2928
Commit: d963454094e50af4aa7db4dbefe90652cd768a92
Parents: c9ae815
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:42:59 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt      |  3 +++
 .../hdfs/tools/DelegationTokenFetcher.java       | 19 +++++++++++--------
 .../hdfs/tools/TestDelegationTokenFetcher.java   | 15 +++++++++++++++
 3 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9634540/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ea67ea1..b065f98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -338,6 +338,9 @@ Trunk (Unreleased)
 
     HDFS-8412. Fix the test failures in HTTPFS. (umamahesh)
 
+    HDFS-8627. NPE thrown if unable to fetch token from Namenode
+    (J.Andreina via vinayakumarb)
+
 Release 2.8.0 - UNRELEASED
 
   NEW FEATURES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9634540/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 6376243..803402d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -176,14 +176,17 @@ public class DelegationTokenFetcher {
                                   final String renewer, final Path tokenFile)
           throws IOException {
     Token<?> token = fs.getDelegationToken(renewer);
-
-    Credentials cred = new Credentials();
-    cred.addToken(token.getKind(), token);
-    cred.writeTokenStorageFile(tokenFile, conf);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Fetched token " + fs.getUri() + " for " + token.getService()
-              + " into " + tokenFile);
+    if (null != token) {
+      Credentials cred = new Credentials();
+      cred.addToken(token.getKind(), token);
+      cred.writeTokenStorageFile(tokenFile, conf);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Fetched token " + fs.getUri() + " for " +
+            token.getService() + " into " + tokenFile);
+      }
+    } else {
+      System.err.println("ERROR: Failed to fetch token from " + fs.getUri());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9634540/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
index b4cf1c0..ab3933b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
@@ -90,4 +90,19 @@ public class TestDelegationTokenFetcher {
     DelegationTokenFetcher.cancelTokens(conf, p);
     Assert.assertEquals(testToken, FakeRenewer.getLastCanceled());
   }
+
+  /**
+   * If token returned is null, saveDelegationToken should not
+   * throw nullPointerException
+   */
+  @Test
+  public void testReturnedTokenIsNull() throws Exception {
+    WebHdfsFileSystem fs = mock(WebHdfsFileSystem.class);
+    doReturn(null).when(fs).getDelegationToken(anyString());
+    Path p = new Path(f.getRoot().getAbsolutePath(), tokenFile);
+    DelegationTokenFetcher.saveDelegationToken(conf, fs, null, p);
+    // When Token returned is null, TokenFile should not exist
+    Assert.assertFalse(p.getFileSystem(conf).exists(p));
+
+  }
 }


[14/48] hadoop git commit: HDFS-8579. Update HDFS usage with missing options (Contributed by J.Andreina)

Posted by zj...@apache.org.
HDFS-8579. Update HDFS usage with missing options (Contributed by J.Andreina)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a943699d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a943699d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a943699d

Branch: refs/heads/YARN-2928
Commit: a943699dce1797a148c8f0c72f5dffe6af92a777
Parents: 4e39b96
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:00:22 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:55 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt       | 3 +++
 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 +
 2 files changed, 4 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a943699d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3535f90..ea67ea1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -983,6 +983,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Brahma
     Reddy Battula via Arpit Agarwal)
 
+    HDFS-8579. Update HDFS usage with missing options
+    (J.Andreina via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a943699d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index f464261..2e09cef 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -26,6 +26,7 @@ function hadoop_usage
   echo "                       Hadoop jar and the required libraries"
   echo "  crypto               configure HDFS encryption zones"
   echo "  datanode             run a DFS datanode"
+  echo "  debug                run a Debug Admin to execute HDFS debug commands"
   echo "  dfs                  run a filesystem command on the file system"
   echo "  dfsadmin             run a DFS admin client"
   echo "  fetchdt              fetch a delegation token from the NameNode"


[41/48] hadoop git commit: HADOOP-12173. NetworkTopology::add calls toString always. Contributed by Inigo Goiri

Posted by zj...@apache.org.
HADOOP-12173. NetworkTopology::add calls toString always. Contributed by Inigo Goiri


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/513eb8b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/513eb8b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/513eb8b6

Branch: refs/heads/YARN-2928
Commit: 513eb8b67659b96510051a65faca7a55164bb291
Parents: 0823b93
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Jul 2 21:39:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:01 2015 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/hadoop/net/NetworkTopology.java      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/513eb8b6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index c60cc0b..63b6763 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -396,14 +396,13 @@ public class NetworkTopology {
     int newDepth = NodeBase.locationToDepth(node.getNetworkLocation()) + 1;
     netlock.writeLock().lock();
     try {
-      String oldTopoStr = this.toString();
       if( node instanceof InnerNode ) {
         throw new IllegalArgumentException(
           "Not allow to add an inner node: "+NodeBase.getPath(node));
       }
       if ((depthOfAllLeaves != -1) && (depthOfAllLeaves != newDepth)) {
         LOG.error("Error: can't add leaf node " + NodeBase.getPath(node) +
-            " at depth " + newDepth + " to topology:\n" + oldTopoStr);
+            " at depth " + newDepth + " to topology:\n" + this.toString());
         throw new InvalidTopologyException("Failed to add " + NodeBase.getPath(node) +
             ": You cannot have a rack and a non-rack node at the same " +
             "level of the network topology.");