You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 07:01:43 UTC
svn commit: r1079263 - in
/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system:
org/apache/hadoop/mapred/gridmix/test/system/ resources/
Author: omalley
Date: Tue Mar 8 06:01:43 2011
New Revision: 1079263
URL: http://svn.apache.org/viewvc?rev=1079263&view=rev
Log:
commit e67c34006222669a2e4cb35176cf28aed949c245
Author: Vinay Kumar Thota <vi...@yahoo-inc.com>
Date: Thu Feb 10 04:39:11 2011 -0800
3964674 from
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz (with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz (with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz (with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz (with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz (with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz (with props)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java?rev=1079263&r1=1079262&r2=1079263&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java Tue Mar 8 06:01:43 2011
@@ -28,7 +28,7 @@ import org.apache.hadoop.util.ToolRunner
import org.apache.hadoop.mapred.gridmix.Gridmix;
import org.apache.hadoop.conf.Configuration;
import java.util.Date;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.net.URI;
@@ -48,6 +48,9 @@ import org.apache.hadoop.test.system.Pro
*/
public class UtilsForGridmix {
private static final Log LOG = LogFactory.getLog(UtilsForGridmix.class);
+ private static final Path DEFAULT_TRACES_PATH =
+ new Path(System.getProperty("user.dir") +
+ "/src/test/system/resources/");
/**
* cleanup the folder or file.
@@ -320,4 +323,139 @@ public class UtilsForGridmix {
}
return pud;
}
+
+ /**
+ * Gives the list of paths for MR traces against different time
+ * intervals.It fetches only the paths which followed the below
+ * file convention.
+ * Syntax : <FileName>_<TimeIntervals>.json.gz
+ * There is a restriction in a file and user has to
+ * follow the below convention for time interval.
+ * Syntax: <numeric>[m|h|d]
+ * e.g : for 10 minutes trace should specify 10m,
+ * same way for 1 hour traces should specify 1h,
+ * for 1 day traces should specify 1d.
+ *
+ * @param conf - cluster configuration.
+ * @return - list of MR paths as key/value pair based on time interval.
+ * @throws IOException - if an I/O error occurs.
+ */
+ public static Map<String, String> getMRTraces(Configuration conf)
+ throws IOException {
+ return getMRTraces(conf, DEFAULT_TRACES_PATH);
+ }
+
+ /**
+ * It gives the list of paths for MR traces against different time
+ * intervals. It fetches only the paths which followed the below
+ * file convention.
+ * Syntax : <FileNames>_<TimeInterval>.json.gz
+ * There is a restriction in a file and user has to follow the
+ * below convention for time interval.
+ * Syntax: <numeric>[m|h|d]
+ * e.g : for 10 minutes trace should specify 10m,
+ * same way for 1 hour traces should specify 1h,
+ * for 1 day traces should specify 1d.
+ *
+ * @param conf - cluster configuration object.
+ * @param tracesPath - MR traces path.
+ * @return - list of MR paths as key/value pair based on time interval.
+ * @throws IOException - If an I/O error occurs.
+ */
+ public static Map<String,String> getMRTraces(Configuration conf,
+ Path tracesPath) throws IOException {
+ Map <String, String> jobTraces = new HashMap <String, String>();
+ final FileSystem fs = FileSystem.getLocal(conf);
+ final FileStatus fstat[] = fs.listStatus(tracesPath);
+ for (FileStatus fst : fstat) {
+ final String fileName = fst.getPath().getName();
+ if (fileName.endsWith("m.json.gz") ||
+ fileName.endsWith("h.json.gz") ||
+ fileName.endsWith("d.json.gz")) {
+ jobTraces.put(fileName.substring(fileName.indexOf("_") + 1,
+ fileName.indexOf(".json.gz")), fst.getPath().toString());
+ }
+ }
+ if (jobTraces.size() == 0) {
+ LOG.error("No traces found in " + tracesPath.toString() + " path.");
+ throw new IOException("No traces found in " +
+ tracesPath.toString() + " path.");
+ }
+ return jobTraces;
+ }
+
+ /**
+ * It list the all the MR traces path irrespective of time.
+ * @param conf - cluster configuration.
+ * @param tracesPath - MR traces path
+ * @return - MR paths as a list.
+ * @throws IOException - if an I/O error occurs.
+ */
+ public static List<String> listMRTraces(Configuration conf,
+ Path tracesPath) throws IOException {
+ List<String> jobTraces = new ArrayList<String>();
+ final FileSystem fs = FileSystem.getLocal(conf);
+ final FileStatus fstat[] = fs.listStatus(tracesPath);
+ for (FileStatus fst : fstat) {
+ jobTraces.add(fst.getPath().toString());
+ }
+ if (jobTraces.size() == 0) {
+ LOG.error("No traces found in " + tracesPath.toString() + " path.");
+ throw new IOException("No traces found in " +
+ tracesPath.toString() + " path.");
+ }
+ return jobTraces;
+ }
+
+ /**
+ * It list the all the MR traces path irrespective of time.
+ * @param conf - cluster configuration.
+ * @param tracesPath - MR traces path
+ * @return - MR paths as a list.
+ * @throws IOException - if an I/O error occurs.
+ */
+ public static List<String> listMRTraces(Configuration conf)
+ throws IOException {
+ return listMRTraces(conf, DEFAULT_TRACES_PATH);
+ }
+
+ /**
+ * Gives the list of MR traces for given time interval.
+ * The time interval should be following convention.
+ * Syntax : <numeric>[m|h|d]
+ * e.g : 10m or 1h or 2d etc.
+ * @param conf - cluster configuration
+ * @param timeInterval - trace time interval.
+ * @param tracesPath - MR traces Path.
+ * @return - MR paths as a list for a given time interval.
+ * @throws IOException - If an I/O error occurs.
+ */
+ public static List<String> listMRTracesByTime(Configuration conf,
+ String timeInterval, Path tracesPath) throws IOException {
+ List<String> jobTraces = new ArrayList<String>();
+ final FileSystem fs = FileSystem.getLocal(conf);
+ final FileStatus fstat[] = fs.listStatus(tracesPath);
+ for (FileStatus fst : fstat) {
+ final String fileName = fst.getPath().getName();
+ if (fileName.indexOf(timeInterval) >= 0) {
+ jobTraces.add(fst.getPath().toString());
+ }
+ }
+ return jobTraces;
+ }
+
+ /**
+ * Gives the list of MR traces for given time interval.
+ * The time interval should be following convention.
+ * Syntax : <numeric>[m|h|d]
+ * e.g : 10m or 1h or 2d etc.
+ * @param conf - cluster configuration
+ * @param timeInterval - trace time interval.
+ * @return - MR paths as a list for a given time interval.
+ * @throws IOException - If an I/O error occurs.
+ */
+ public static List<String> listMRTracesByTime(Configuration conf,
+ String timeInterval) throws IOException {
+ return listMRTracesByTime(conf, timeInterval, DEFAULT_TRACES_PATH);
+ }
}
Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_10m.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_12m.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_1m.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_3m.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_5m.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz?rev=1079263&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/trace_7m.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream