svn commit: r981996 - in /hadoop/mapreduce/trunk: ./ src/contrib/mumak/src/java/org/apache/hadoop/mapred/ src/contrib/mumak/src/test/org/apache/hadoop/mapred/

[ma...@apache.org]

Author: mahadev
Date: Tue Aug  3 18:23:34 2010
New Revision: 981996

URL: http://svn.apache.org/viewvc?rev=981996&view=rev
Log:
MAPREDUCE-1834. TestSimulatorDeterministicReplay timesout on trunk. (Hong Tang via mahadev)

Added:
    hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/FakeConcurrentHashMap.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/DeterministicCollectionAspects.aj
    hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=981996&r1=981995&r2=981996&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Aug  3 18:23:34 2010
@@ -105,6 +105,9 @@ Trunk (unreleased changes)
     with delegation tokens (on the client side). 
     (Jitendra Pandey and Devaraj Das via ddas)
 
+    MAPREDUCE-1834. TestSimulatorDeterministicReplay timesout on trunk.
+    (Hong Tang via mahadev)
+
   OPTIMIZATIONS
 
     MAPREDUCE-1354. Enhancements to JobTracker for better performance and

Modified: hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/DeterministicCollectionAspects.aj
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/DeterministicCollectionAspects.aj?rev=981996&r1=981995&r2=981996&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/DeterministicCollectionAspects.aj (original)
+++ hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/DeterministicCollectionAspects.aj Tue Aug  3 18:23:34 2010
@@ -21,6 +21,7 @@ import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.concurrent.ConcurrentHashMap;
 
 // HashSet and HashMap do not guarantee that the oder of iteration is 
 // determinstic. We need the latter for the deterministic replay of 
@@ -42,4 +43,8 @@ public privileged aspect DeterministicCo
   HashMap around() : call(HashMap.new()) {
     return new LinkedHashMap();
   }
+
+  ConcurrentHashMap around() : call(ConcurrentHashMap.new()) {
+    return new FakeConcurrentHashMap();
+  }
 }

Added: hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/FakeConcurrentHashMap.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/FakeConcurrentHashMap.java?rev=981996&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/FakeConcurrentHashMap.java (added)
+++ hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/FakeConcurrentHashMap.java Tue Aug  3 18:23:34 2010
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A fake ConcurrentHashMap implementation that maintains the insertion order of
+ * entries when traversing through iterator. The class is to support
+ * deterministic replay of Mumak and not meant to be used as a ConcurrentHashMap
+ * replacement with multiple threads.
+ */
+public class FakeConcurrentHashMap<K, V> extends ConcurrentHashMap<K, V> {
+  private final Map<K, V> map;
+  
+  public FakeConcurrentHashMap() {
+    map = new LinkedHashMap<K, V>();
+  }
+  
+  @Override
+  public V putIfAbsent(K key, V value) {
+    if (!containsKey(key)) {
+      return put(key, value);
+    } else {
+      return get(key);
+    }
+  }
+
+  @Override
+  public boolean remove(Object key, Object value) {
+    if (!containsKey(key)) return false;
+    Object oldValue = get(key);
+    if ((oldValue == null) ? value == null : oldValue.equals(value)) {
+      remove(key);
+      return true;
+    }
+    return false;
+  }
+
+  @Override
+  public V replace(K key, V value) {
+    if (containsKey(key)) {
+      return put(key, value);
+    } else {
+      return null;
+    }
+  }
+
+  @Override
+  public boolean replace(K key, V oldValue, V newValue) {
+    if (!containsKey(key)) return false;
+    Object origValue = get(key);
+    if ((origValue == null) ? oldValue == null : origValue.equals(oldValue)) {
+      put(key, newValue);
+      return true;
+    }
+    return false;
+  }
+
+  @Override
+  public void clear() {
+    map.clear();
+  }
+
+  @Override
+  public boolean containsKey(Object key) {
+    return map.containsKey(key);
+  }
+
+  @Override
+  public boolean containsValue(Object value) {
+    return map.containsValue(value);
+  }
+
+  @Override
+  public Set<Map.Entry<K, V>> entrySet() {
+    return map.entrySet();
+  }
+
+  @Override
+  public V get(Object key) {
+    return map.get(key);
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return map.isEmpty();
+  }
+
+  @Override
+  public Set<K> keySet() {
+    return map.keySet();
+  }
+
+  @Override
+  public V put(K key, V value) {
+    return map.put(key, value);
+  }
+
+  @Override
+  public void putAll(Map<? extends K, ? extends V> t) {
+    map.putAll(t);
+  }
+
+  @Override
+  public V remove(Object key) {
+    return map.remove(key);
+  }
+
+  @Override
+  public int size() {
+    return map.size();
+  }
+
+  @Override
+  public Collection<V> values() {
+    return map.values();
+  }
+}

Modified: hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java?rev=981996&r1=981995&r2=981996&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java Tue Aug  3 18:23:34 2010
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.mapred;
 
-import java.io.IOException;
-
 import junit.framework.Assert;
 
 import org.apache.commons.logging.Log;
@@ -27,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
 import org.junit.Test;
 
@@ -50,23 +49,33 @@ public class TestSimulatorDeterministicR
   
   void compareLogDirs(String dir1, String dir2) {
     try {
-      Runtime runtime = Runtime.getRuntime();
-      Process process = runtime.exec("diff -r /dev/null /dev/null");
-      process.waitFor();
-      // If there is no diff available, we skip the test and end up in 
-      // the catch block
-      // Make sure diff understands the -r option
-      if (process.exitValue() != 0) {
-        LOG.warn("diff -r is not working, skipping the test");
+      try {
+        // If there is no diff available, we skip the test and end up in
+        // the catch block
+        // Make sure diff understands the -r option
+        ShellCommandExecutor executor = new ShellCommandExecutor(new String[] {
+            "diff", "-r", "-q", "/dev/null", "/dev/null" });
+        executor.execute();
+        if (executor.getExitCode() != 0) {
+          LOG.warn("diff -r -q is not working, skipping the test");
+          return;
+        }
+      } catch (Exception e) {
+        LOG.warn("diff -r -q is not working, skipping the test", e);
         return;
       }
+
       // Run the real comparison
-      process = runtime.exec("diff -r " + dir1 + " " + dir2);
-      process.waitFor();
-      Assert.assertEquals("Job history logs differ, diff returned", 
-                          0, process.exitValue());
+      ShellCommandExecutor executor = 
+    	  new ShellCommandExecutor(new String[] {"diff", "-r", "-q", dir1, dir2});
+      executor.execute();
+      Assert.assertEquals("Job history logs differ, diff returned",
+    		  0, executor.getExitCode());
     } catch (Exception e) {
       LOG.warn("Exception while diffing: " + e);
+      Assert.fail(String.format(
+    		  "Exception while diffing %s and %s. Exception - %s",
+    		  dir1, dir2, e));
     }                        
   }