You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 04:38:23 UTC
svn commit: r1077079 [10/11] - in
/hadoop/common/branches/branch-0.20-security-patches: ./ src/contrib/
src/contrib/gridmix/ src/contrib/gridmix/ivy/ src/contrib/gridmix/src/
src/contrib/gridmix/src/java/ src/contrib/gridmix/src/java/org/
src/contrib/g...
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/Histogram.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/Histogram.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/Histogram.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/Histogram.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * {@link Histogram} represents an ordered summary of a sequence of {@code long}
+ * s which can be queried to produce a discrete approximation of its cumulative
+ * distribution function
+ *
+ */
+class Histogram implements Iterable<Map.Entry<Long, Long>> {
+ private TreeMap<Long, Long> content = new TreeMap<Long, Long>();
+
+ private String name;
+
+ private long totalCount;
+
+ public Histogram() {
+ this("(anonymous)");
+ }
+
+ public Histogram(String name) {
+ super();
+
+ this.name = name;
+
+ totalCount = 0L;
+ }
+
+ public void dump(PrintStream stream) {
+ stream.print("dumping Histogram " + name + ":\n");
+
+ Iterator<Map.Entry<Long, Long>> iter = iterator();
+
+ while (iter.hasNext()) {
+ Map.Entry<Long, Long> ent = iter.next();
+
+ stream.print("val/count pair: " + (long) ent.getKey() + ", "
+ + (long) ent.getValue() + "\n");
+ }
+
+ stream.print("*** end *** \n");
+ }
+
+ public Iterator<Map.Entry<Long, Long>> iterator() {
+ return content.entrySet().iterator();
+ }
+
+ public long get(long key) {
+ Long result = content.get(key);
+
+ return result == null ? 0 : result;
+ }
+
+ public long getTotalCount() {
+ return totalCount;
+ }
+
+ public void enter(long value) {
+ Long existingValue = content.get(value);
+
+ if (existingValue == null) {
+ content.put(value, 1L);
+ } else {
+ content.put(value, existingValue + 1L);
+ }
+
+ ++totalCount;
+ }
+
+ /**
+ * Produces a discrete approximation of the CDF. The user provides the points
+ * on the {@code Y} axis he wants, and we give the corresponding points on the
+ * {@code X} axis, plus the minimum and maximum from the data.
+ *
+ * @param scale
+ * the denominator applied to every element of buckets. For example,
+ * if {@code scale} is {@code 1000}, a {@code buckets} element of 500
+ * will specify the median in that output slot.
+ * @param buckets
+ * an array of int, all less than scale and each strictly greater
+ * than its predecessor if any. We don't check these requirements.
+ * @return a {@code long[]}, with two more elements than {@code buckets} has.
+ * The first resp. last element is the minimum resp. maximum value
+ * that was ever {@code enter}ed. The rest of the elements correspond
+ * to the elements of {@code buckets} and carry the first element
+ * whose rank is no less than {@code #content elements * scale /
+ * bucket}.
+ *
+ */
+ public long[] getCDF(int scale, int[] buckets) {
+ if (totalCount == 0) {
+ return null;
+ }
+
+ long[] result = new long[buckets.length + 2];
+
+ // fill in the min and the max
+ result[0] = content.firstEntry().getKey();
+
+ result[buckets.length + 1] = content.lastEntry().getKey();
+
+ Iterator<Map.Entry<Long, Long>> iter = content.entrySet().iterator();
+ long cumulativeCount = 0;
+ int bucketCursor = 0;
+
+
+ // Loop invariant: the item at buckets[bucketCursor] can still be reached
+ // from iter, and the number of logged elements no longer available from
+ // iter is cumulativeCount.
+ //
+ // cumulativeCount/totalCount is therefore strictly less than
+ // buckets[bucketCursor]/scale .
+
+ while (iter.hasNext()) {
+ long targetCumulativeCount = buckets[bucketCursor] * totalCount / scale;
+
+ Map.Entry<Long, Long> elt = iter.next();
+
+ cumulativeCount += elt.getValue();
+
+ while (cumulativeCount >= targetCumulativeCount) {
+ result[bucketCursor + 1] = elt.getKey();
+
+ ++bucketCursor;
+
+ if (bucketCursor < buckets.length) {
+ targetCumulativeCount = buckets[bucketCursor] * totalCount / scale;
+ } else {
+ break;
+ }
+ }
+
+ if (bucketCursor == buckets.length) {
+ break;
+ }
+ }
+
+ return result;
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStory.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStory.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStory.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants.Values;
+
+/**
+ * {@link JobStory} represents the runtime information available for a
+ * completed Map-Reduce job.
+ */
+public interface JobStory {
+
+ /**
+ * Get the {@link JobConf} for the job.
+ * @return the <code>JobConf</code> for the job
+ */
+ public JobConf getJobConf();
+
+ /**
+ * Get the job name.
+ * @return the job name
+ */
+ public String getName();
+
+ /**
+ * Get the job ID
+ * @return the job ID
+ */
+ public JobID getJobID();
+
+ /**
+ * Get the user who ran the job.
+ * @return the user who ran the job
+ */
+ public String getUser();
+
+ /**
+ * Get the job submission time.
+ * @return the job submission time
+ */
+ public long getSubmissionTime();
+
+ /**
+ * Get the number of maps in the {@link JobStory}.
+ * @return the number of maps in the <code>Job</code>
+ */
+ public int getNumberMaps();
+
+ /**
+ * Get the number of reduce in the {@link JobStory}.
+ * @return the number of reduces in the <code>Job</code>
+ */
+ public int getNumberReduces();
+
+ /**
+ * Get the input splits for the job.
+ * @return the input splits for the job
+ */
+ public InputSplit[] getInputSplits();
+
+ /**
+ * Get {@link TaskInfo} for a given task.
+ * @param taskType {@link TaskType} of the task
+ * @param taskNumber Partition number of the task
+ * @return the <code>TaskInfo</code> for the given task
+ */
+ public TaskInfo getTaskInfo(TaskType taskType, int taskNumber);
+
+ /**
+ * Get {@link TaskAttemptInfo} for a given task-attempt, without regard to
+ * impact of locality (e.g. not needed to make scheduling decisions).
+ * @param taskType {@link TaskType} of the task-attempt
+ * @param taskNumber Partition number of the task-attempt
+ * @param taskAttemptNumber Attempt number of the task
+ * @return the <code>TaskAttemptInfo</code> for the given task-attempt
+ */
+ public TaskAttemptInfo getTaskAttemptInfo(TaskType taskType,
+ int taskNumber,
+ int taskAttemptNumber);
+
+ /**
+ * Get {@link TaskAttemptInfo} for a given task-attempt, considering impact
+ * of locality.
+ * @param taskNumber Partition number of the task-attempt
+ * @param taskAttemptNumber Attempt number of the task
+ * @param locality Data locality of the task as scheduled in simulation
+ * @return the <code>TaskAttemptInfo</code> for the given task-attempt
+ */
+ public TaskAttemptInfo
+ getMapTaskAttemptInfoAdjusted(int taskNumber,
+ int taskAttemptNumber,
+ int locality);
+
+ /**
+ * Get the outcome of the job execution.
+ * @return The outcome of the job execution.
+ */
+ public Values getOutcome();
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStoryProducer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStoryProducer.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStoryProducer.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobStoryProducer.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * {@link JobStoryProducer} produces the sequence of {@link JobStory}'s.
+ */
+public interface JobStoryProducer extends Closeable {
+ /**
+ * Get the next job.
+ * @return The next job. Or null if no more job is available.
+ * @throws IOException
+ */
+ JobStory getNextJob() throws IOException;
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobTraceReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobTraceReader.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobTraceReader.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JobTraceReader.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Reading JSON-encoded job traces and produce {@link LoggedJob} instances.
+ */
+public class JobTraceReader extends JsonObjectMapperParser<LoggedJob> {
+ /**
+ * Constructor.
+ *
+ * @param path
+ * Path to the JSON trace file, possibly compressed.
+ * @param conf
+ * @throws IOException
+ */
+ public JobTraceReader(Path path, Configuration conf) throws IOException {
+ super(path, LoggedJob.class, conf);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param input
+ * The input stream for the JSON trace.
+ */
+ public JobTraceReader(InputStream input) throws IOException {
+ super(input, LoggedJob.class);
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * A simple wrapper for parsing JSON-encoded data using ObjectMapper.
+ * @param <T> The (base) type of the object(s) to be parsed by this parser.
+ */
+class JsonObjectMapperParser<T> implements Closeable {
+ private final ObjectMapper mapper;
+ private final Class<? extends T> clazz;
+ private final JsonParser jsonParser;
+ private final Decompressor decompressor;
+
+ /**
+ * Constructor.
+ *
+ * @param path
+ * Path to the JSON data file, possibly compressed.
+ * @param conf
+ * @throws IOException
+ */
+ public JsonObjectMapperParser(Path path, Class<? extends T> clazz,
+ Configuration conf) throws IOException {
+ mapper = new ObjectMapper();
+ mapper.configure(
+ DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
+ this.clazz = clazz;
+ FileSystem fs = path.getFileSystem(conf);
+ CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path);
+ InputStream input;
+ if (codec == null) {
+ input = fs.open(path);
+ decompressor = null;
+ } else {
+ FSDataInputStream fsdis = fs.open(path);
+ decompressor = CodecPool.getDecompressor(codec);
+ input = codec.createInputStream(fsdis, decompressor);
+ }
+ jsonParser = mapper.getJsonFactory().createJsonParser(input);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param input
+ * The input stream for the JSON data.
+ */
+ public JsonObjectMapperParser(InputStream input, Class<? extends T> clazz)
+ throws IOException {
+ mapper = new ObjectMapper();
+ mapper.configure(
+ DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
+ this.clazz = clazz;
+ decompressor = null;
+ jsonParser = mapper.getJsonFactory().createJsonParser(input);
+ }
+
+ /**
+ * Get the next object from the trace.
+ *
+ * @return The next instance of the object. Or null if we reach the end of
+ * stream.
+ * @throws IOException
+ */
+ public T getNext() throws IOException {
+ try {
+ return mapper.readValue(jsonParser, clazz);
+ } catch (EOFException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ jsonParser.close();
+ } finally {
+ if (decompressor != null) {
+ CodecPool.returnDecompressor(decompressor);
+ }
+ }
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LogRecordType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LogRecordType.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LogRecordType.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LogRecordType.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools.rumen;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Iterator;
+
+class LogRecordType {
+ static Map<String, LogRecordType> internees = new HashMap<String, LogRecordType>();
+
+ final String name;
+
+ final int index;
+
+ private LogRecordType(String name) {
+ super();
+
+ this.name = name;
+
+ index = internees.size();
+ }
+
+ static LogRecordType intern(String typeName) {
+ LogRecordType result = internees.get(typeName);
+
+ if (result == null) {
+ result = new LogRecordType(typeName);
+
+ internees.put(typeName, result);
+ }
+
+ return result;
+ }
+
+ static LogRecordType internSoft(String typeName) {
+ return internees.get(typeName);
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ static String[] lineTypes() {
+ Iterator<Map.Entry<String, LogRecordType>> iter = internees.entrySet()
+ .iterator();
+
+ String[] result = new String[internees.size()];
+
+ for (int i = 0; i < internees.size(); ++i) {
+ result[i] = iter.next().getKey();
+ }
+
+ return result;
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedDiscreteCDF.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedDiscreteCDF.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedDiscreteCDF.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedDiscreteCDF.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A {@link LoggedDiscreteCDF} is a discrete approximation of a cumulative
+ * distribution function, with this class set up to meet the requirements of the
+ * Jackson JSON parser/generator.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedDiscreteCDF implements DeepCompare {
+ /**
+ * The number of values this CDF is built on
+ */
+ long numberValues = -1L;
+ /**
+ * The least {@code X} value
+ */
+ long minimum = Long.MIN_VALUE;
+ /**
+ * The coordinates of the bulk of the CDF
+ */
+ List<LoggedSingleRelativeRanking> rankings = new ArrayList<LoggedSingleRelativeRanking>();
+ /**
+ * The greatest {@code X} value
+ */
+ long maximum = Long.MAX_VALUE;
+
+ void setCDF(Histogram data, int[] steps, int modulus) {
+
+ numberValues = data.getTotalCount();
+ long[] CDF = data.getCDF(modulus, steps);
+
+ if (CDF != null) {
+ minimum = CDF[0];
+ maximum = CDF[CDF.length - 1];
+
+ rankings = new ArrayList<LoggedSingleRelativeRanking>();
+
+ for (int i = 1; i < CDF.length - 1; ++i) {
+ LoggedSingleRelativeRanking srr = new LoggedSingleRelativeRanking();
+
+ srr.setRelativeRanking(((double) steps[i - 1]) / modulus);
+ srr.setDatum(CDF[i]);
+
+ rankings.add(srr);
+ }
+ }
+ }
+
+ public long getMinimum() {
+ return minimum;
+ }
+
+ void setMinimum(long minimum) {
+ this.minimum = minimum;
+ }
+
+ public List<LoggedSingleRelativeRanking> getRankings() {
+ return rankings;
+ }
+
+ void setRankings(List<LoggedSingleRelativeRanking> rankings) {
+ this.rankings = rankings;
+ }
+
+ public long getMaximum() {
+ return maximum;
+ }
+
+ void setMaximum(long maximum) {
+ this.maximum = maximum;
+ }
+
+ public long getNumberValues() {
+ return numberValues;
+ }
+
+ void setNumberValues(long numberValues) {
+ this.numberValues = numberValues;
+ }
+
+ private void compare1(long c1, long c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(List<LoggedSingleRelativeRanking> c1,
+ List<LoggedSingleRelativeRanking> c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || c1.size() != c2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+
+ for (int i = 0; i < c1.size(); ++i) {
+ c1.get(i).deepCompare(c2.get(i), new TreePath(loc, eltname, i));
+ }
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedDiscreteCDF)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedDiscreteCDF other = (LoggedDiscreteCDF) comparand;
+
+ compare1(numberValues, other.numberValues, loc, "numberValues");
+
+ compare1(minimum, other.minimum, loc, "minimum");
+ compare1(maximum, other.maximum, loc, "maximum");
+
+ compare1(rankings, other.rankings, loc, "rankings");
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,586 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
+/**
+ * A {@link LoggedDiscreteCDF} is a representation of an hadoop job, with the
+ * details of this class set up to meet the requirements of the Jackson JSON
+ * parser/generator.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedJob implements DeepCompare {
+ public enum JobType {
+ JAVA, PIG, STREAMING, PIPES, OVERALL
+ };
+
+ public enum JobPriority {
+ VERY_LOW, LOW, NORMAL, HIGH, VERY_HIGH
+ };
+
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ String jobID;
+ String user;
+ long computonsPerMapInputByte = -1L;
+ long computonsPerMapOutputByte = -1L;
+ long computonsPerReduceInputByte = -1L;
+ long computonsPerReduceOutputByte = -1L;
+ long submitTime = -1L;
+ long launchTime = -1L;
+ long finishTime = -1L;
+
+ int heapMegabytes = -1;
+ int totalMaps = -1;
+ int totalReduces = -1;
+ Pre21JobHistoryConstants.Values outcome = null;
+ JobType jobtype = JobType.JAVA;
+ JobPriority priority = JobPriority.NORMAL;
+
+ List<String> directDependantJobs = new ArrayList<String>();
+ List<LoggedTask> mapTasks = new ArrayList<LoggedTask>();
+ List<LoggedTask> reduceTasks = new ArrayList<LoggedTask>();
+ List<LoggedTask> otherTasks = new ArrayList<LoggedTask>();
+
+ // There are CDFs for each level of locality -- most local first
+ ArrayList<LoggedDiscreteCDF> successfulMapAttemptCDFs;
+ // There are CDFs for each level of locality -- most local first
+ ArrayList<LoggedDiscreteCDF> failedMapAttemptCDFs;
+
+ LoggedDiscreteCDF successfulReduceAttemptCDF;
+ LoggedDiscreteCDF failedReduceAttemptCDF;
+
+ String queue = null;
+
+ String jobName = null;
+
+ int clusterMapMB = -1;
+ int clusterReduceMB = -1;
+ int jobMapMB = -1;
+ int jobReduceMB = -1;
+
+ long relativeTime = 0;
+
+ double[] mapperTriesToSucceed;
+ double failedMapperFraction; // !!!!!
+
+ LoggedJob() {
+
+ }
+
+ LoggedJob(String jobID) {
+ super();
+
+ setJobID(jobID);
+ }
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ void setUser(String user) {
+ this.user = user;
+ }
+
+ public String getJobID() {
+ return jobID;
+ }
+
+ void setJobID(String jobID) {
+ this.jobID = jobID;
+ }
+
+ public JobPriority getPriority() {
+ return priority;
+ }
+
+ void setPriority(JobPriority priority) {
+ this.priority = priority;
+ }
+
+ public long getComputonsPerMapInputByte() {
+ return computonsPerMapInputByte;
+ }
+
+ void setComputonsPerMapInputByte(long computonsPerMapInputByte) {
+ this.computonsPerMapInputByte = computonsPerMapInputByte;
+ }
+
+ public long getComputonsPerMapOutputByte() {
+ return computonsPerMapOutputByte;
+ }
+
+ void setComputonsPerMapOutputByte(long computonsPerMapOutputByte) {
+ this.computonsPerMapOutputByte = computonsPerMapOutputByte;
+ }
+
+ public long getComputonsPerReduceInputByte() {
+ return computonsPerReduceInputByte;
+ }
+
+ void setComputonsPerReduceInputByte(long computonsPerReduceInputByte) {
+ this.computonsPerReduceInputByte = computonsPerReduceInputByte;
+ }
+
+ public long getComputonsPerReduceOutputByte() {
+ return computonsPerReduceOutputByte;
+ }
+
+ void setComputonsPerReduceOutputByte(long computonsPerReduceOutputByte) {
+ this.computonsPerReduceOutputByte = computonsPerReduceOutputByte; // !!!!!
+ }
+
+ public long getSubmitTime() {
+ return submitTime;
+ }
+
+ void setSubmitTime(long submitTime) {
+ this.submitTime = submitTime;
+ }
+
+ public long getLaunchTime() {
+ return launchTime;
+ }
+
+ void setLaunchTime(long startTime) {
+ this.launchTime = startTime;
+ }
+
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ public int getHeapMegabytes() {
+ return heapMegabytes;
+ }
+
+ void setHeapMegabytes(int heapMegabytes) {
+ this.heapMegabytes = heapMegabytes;
+ }
+
+ public int getTotalMaps() {
+ return totalMaps;
+ }
+
+ void setTotalMaps(int totalMaps) {
+ this.totalMaps = totalMaps;
+ }
+
+ public int getTotalReduces() {
+ return totalReduces;
+ }
+
+ void setTotalReduces(int totalReduces) {
+ this.totalReduces = totalReduces;
+ }
+
+ public Pre21JobHistoryConstants.Values getOutcome() {
+ return outcome;
+ }
+
+ void setOutcome(Pre21JobHistoryConstants.Values outcome) {
+ this.outcome = outcome;
+ }
+
+ public JobType getJobtype() {
+ return jobtype;
+ }
+
+ void setJobtype(JobType jobtype) {
+ this.jobtype = jobtype;
+ }
+
+ public List<String> getDirectDependantJobs() {
+ return directDependantJobs;
+ }
+
+ void setDirectDependantJobs(List<String> directDependantJobs) {
+ this.directDependantJobs = directDependantJobs;
+ }
+
+ public List<LoggedTask> getMapTasks() {
+ return mapTasks;
+ }
+
+ void setMapTasks(List<LoggedTask> mapTasks) {
+ this.mapTasks = mapTasks;
+ }
+
+ public List<LoggedTask> getReduceTasks() {
+ return reduceTasks;
+ }
+
+ void setReduceTasks(List<LoggedTask> reduceTasks) {
+ this.reduceTasks = reduceTasks;
+ }
+
+ public List<LoggedTask> getOtherTasks() {
+ return otherTasks;
+ }
+
+ void setOtherTasks(List<LoggedTask> otherTasks) {
+ this.otherTasks = otherTasks;
+ }
+
+ public ArrayList<LoggedDiscreteCDF> getSuccessfulMapAttemptCDFs() {
+ return successfulMapAttemptCDFs;
+ }
+
+ void setSuccessfulMapAttemptCDFs(
+ ArrayList<LoggedDiscreteCDF> successfulMapAttemptCDFs) {
+ this.successfulMapAttemptCDFs = successfulMapAttemptCDFs;
+ }
+
+ public ArrayList<LoggedDiscreteCDF> getFailedMapAttemptCDFs() {
+ return failedMapAttemptCDFs;
+ }
+
+ void setFailedMapAttemptCDFs(ArrayList<LoggedDiscreteCDF> failedMapAttemptCDFs) {
+ this.failedMapAttemptCDFs = failedMapAttemptCDFs;
+ }
+
+ public LoggedDiscreteCDF getSuccessfulReduceAttemptCDF() {
+ return successfulReduceAttemptCDF;
+ }
+
+ void setSuccessfulReduceAttemptCDF(
+ LoggedDiscreteCDF successfulReduceAttemptCDF) {
+ this.successfulReduceAttemptCDF = successfulReduceAttemptCDF;
+ }
+
+ public LoggedDiscreteCDF getFailedReduceAttemptCDF() {
+ return failedReduceAttemptCDF;
+ }
+
+ void setFailedReduceAttemptCDF(LoggedDiscreteCDF failedReduceAttemptCDF) {
+ this.failedReduceAttemptCDF = failedReduceAttemptCDF;
+ }
+
+ public double[] getMapperTriesToSucceed() {
+ return mapperTriesToSucceed;
+ }
+
+ void setMapperTriesToSucceed(double[] mapperTriesToSucceed) {
+ this.mapperTriesToSucceed = mapperTriesToSucceed;
+ }
+
+ public double getFailedMapperFraction() {
+ return failedMapperFraction;
+ }
+
+ void setFailedMapperFraction(double failedMapperFraction) {
+ this.failedMapperFraction = failedMapperFraction;
+ }
+
+ public long getRelativeTime() {
+ return relativeTime;
+ }
+
+ void setRelativeTime(long relativeTime) {
+ this.relativeTime = relativeTime;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public int getClusterMapMB() {
+ return clusterMapMB;
+ }
+
+ void setClusterMapMB(int clusterMapMB) {
+ this.clusterMapMB = clusterMapMB;
+ }
+
+ public int getClusterReduceMB() {
+ return clusterReduceMB;
+ }
+
+ void setClusterReduceMB(int clusterReduceMB) {
+ this.clusterReduceMB = clusterReduceMB;
+ }
+
+ public int getJobMapMB() {
+ return jobMapMB;
+ }
+
+ void setJobMapMB(int jobMapMB) {
+ this.jobMapMB = jobMapMB;
+ }
+
+ public int getJobReduceMB() {
+ return jobReduceMB;
+ }
+
+ void setJobReduceMB(int jobReduceMB) {
+ this.jobReduceMB = jobReduceMB;
+ }
+
+ private void compare1(String c1, String c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || !c1.equals(c2)) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(long c1, long c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(Pre21JobHistoryConstants.Values c1,
+ Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(JobType c1, JobType c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(JobPriority c1, JobPriority c2, TreePath loc,
+ String eltname) throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(int c1, int c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(double c1, double c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(double[] c1, double[] c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ TreePath recursePath = new TreePath(loc, eltname);
+
+ if (c1 == null || c2 == null || c1.length != c2.length) {
+ throw new DeepInequalityException(eltname + " miscompared", recursePath);
+ }
+
+ for (int i = 0; i < c1.length; ++i) {
+ if (c1[i] != c2[i]) {
+ throw new DeepInequalityException(eltname + " miscompared",
+ new TreePath(loc, eltname, i));
+ }
+ }
+ }
+
+ private void compare1(DeepCompare c1, DeepCompare c2, TreePath loc,
+ String eltname, int index) throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ TreePath recursePath = new TreePath(loc, eltname, index);
+
+ if (c1 == null || c2 == null) {
+ if (index == -1) {
+ throw new DeepInequalityException(eltname + " miscompared", recursePath);
+ } else {
+ throw new DeepInequalityException(eltname + "[" + index
+ + "] miscompared", recursePath);
+ }
+ }
+
+ c1.deepCompare(c2, recursePath);
+ }
+
+ // I'll treat this as an atomic object type
+ private void compareStrings(List<String> c1, List<String> c2, TreePath loc,
+ String eltname) throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ TreePath recursePath = new TreePath(loc, eltname);
+
+ if (c1 == null || c2 == null || !c1.equals(c2)) {
+ throw new DeepInequalityException(eltname + " miscompared", recursePath);
+ }
+ }
+
+ private void compareLoggedTasks(List<LoggedTask> c1, List<LoggedTask> c2,
+ TreePath loc, String eltname) throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || c1.size() != c2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+
+ for (int i = 0; i < c1.size(); ++i) {
+ c1.get(i).deepCompare(c2.get(i), new TreePath(loc, eltname, i));
+ }
+ }
+
+ private void compareCDFs(List<LoggedDiscreteCDF> c1,
+ List<LoggedDiscreteCDF> c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || c1.size() != c2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+
+ for (int i = 0; i < c1.size(); ++i) {
+ c1.get(i).deepCompare(c2.get(i), new TreePath(loc, eltname, i));
+ }
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedJob)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedJob other = (LoggedJob) comparand;
+
+ compare1(jobID, other.jobID, loc, "jobID");
+ compare1(user, other.user, loc, "user");
+
+ compare1(computonsPerMapInputByte, other.computonsPerMapInputByte, loc,
+ "computonsPerMapInputByte");
+ compare1(computonsPerMapOutputByte, other.computonsPerMapOutputByte, loc,
+ "computonsPerMapOutputByte");
+ compare1(computonsPerReduceInputByte, other.computonsPerReduceInputByte,
+ loc, "computonsPerReduceInputByte");
+ compare1(computonsPerReduceOutputByte, other.computonsPerReduceOutputByte,
+ loc, "computonsPerReduceOutputByte");
+
+ compare1(submitTime, other.submitTime, loc, "submitTime");
+ compare1(launchTime, other.launchTime, loc, "launchTime");
+ compare1(finishTime, other.finishTime, loc, "finishTime");
+
+ compare1(heapMegabytes, other.heapMegabytes, loc, "heapMegabytes");
+
+ compare1(totalMaps, other.totalMaps, loc, "totalMaps");
+ compare1(totalReduces, other.totalReduces, loc, "totalReduces");
+
+ compare1(outcome, other.outcome, loc, "outcome");
+ compare1(jobtype, other.jobtype, loc, "jobtype");
+ compare1(priority, other.priority, loc, "priority");
+
+ compareStrings(directDependantJobs, other.directDependantJobs, loc,
+ "directDependantJobs");
+
+ compareLoggedTasks(mapTasks, other.mapTasks, loc, "mapTasks");
+ compareLoggedTasks(reduceTasks, other.reduceTasks, loc, "reduceTasks");
+ compareLoggedTasks(otherTasks, other.otherTasks, loc, "otherTasks");
+
+ compare1(relativeTime, other.relativeTime, loc, "relativeTime");
+
+ compareCDFs(successfulMapAttemptCDFs, other.successfulMapAttemptCDFs, loc,
+ "successfulMapAttemptCDFs");
+ compareCDFs(failedMapAttemptCDFs, other.failedMapAttemptCDFs, loc,
+ "failedMapAttemptCDFs");
+ compare1(successfulReduceAttemptCDF, other.successfulReduceAttemptCDF, loc,
+ "successfulReduceAttemptCDF", -1);
+ compare1(failedReduceAttemptCDF, other.failedReduceAttemptCDF, loc,
+ "failedReduceAttemptCDF", -1);
+
+ compare1(mapperTriesToSucceed, other.mapperTriesToSucceed, loc,
+ "mapperTriesToSucceed");
+ compare1(failedMapperFraction, other.failedMapperFraction, loc,
+ "failedMapperFraction");
+
+ compare1(queue, other.queue, loc, "queue");
+ compare1(jobName, other.jobName, loc, "jobName");
+
+ compare1(clusterMapMB, other.clusterMapMB, loc, "clusterMapMB");
+ compare1(clusterReduceMB, other.clusterReduceMB, loc, "clusterReduceMB");
+ compare1(jobMapMB, other.jobMapMB, loc, "jobMapMB");
+ compare1(jobReduceMB, other.jobReduceMB, loc, "jobReduceMB");
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
+/**
+ * A {@link LoggedLocation} is a representation of a point in an hierarchical
+ * network, represented as a series of membership names, broadest first.
+ *
+ * For example, if your network has <i>hosts</i> grouped into <i>racks</i>, then
+ * in onecluster you might have a node {@code node1} on rack {@code rack1}. This
+ * would be represented with a ArrayList of two layers, with two {@link String}
+ * s being {@code "rack1"} and {@code "node1"}.
+ *
+ * The details of this class are set up to meet the requirements of the Jackson
+ * JSON parser/generator.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedLocation implements DeepCompare {
+ /**
+ * The full path from the root of the network to the host.
+ *
+ * NOTE that this assumes that the network topology is a tree.
+ */
+ List<String> layers = new ArrayList<String>();
+
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ public List<String> getLayers() {
+ return layers;
+ }
+
+ void setLayers(List<String> layers) {
+ this.layers = layers;
+ }
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
+ // I'll treat this as an atomic object type
+ private void compareStrings(List<String> c1, List<String> c2, TreePath loc,
+ String eltname) throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ TreePath recursePath = new TreePath(loc, eltname);
+
+ if (c1 == null || c2 == null || !c1.equals(c2)) {
+ throw new DeepInequalityException(eltname + " miscompared", recursePath);
+ }
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedLocation)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedLocation other = (LoggedLocation) comparand;
+
+ compareStrings(layers, other.layers, loc, "layers");
+
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
+/**
+ * A {@link LoggedNetworkTopology} represents a tree that in turn represents a
+ * hierarchy of hosts. The current version requires the tree to have all leaves
+ * at the same level.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedNetworkTopology implements DeepCompare {
+ String name;
+ List<LoggedNetworkTopology> children = new ArrayList<LoggedNetworkTopology>();
+
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ public LoggedNetworkTopology() {
+ super();
+ }
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
+ /**
+ * We need this because we have to sort the {@code children} field. That field
+ * is set-valued, but if we sort these fields we ensure that comparisons won't
+ * bogusly fail because the hash table happened to enumerate in a different
+ * order.
+ *
+ */
+ static class TopoSort implements Comparator<LoggedNetworkTopology> {
+ public int compare(LoggedNetworkTopology t1, LoggedNetworkTopology t2) {
+ return t1.name.compareTo(t2.name);
+ }
+ }
+
+ /**
+ * @param hosts
+ * a HashSet of the {@link ParsedHost}
+ * @param name
+ * the name of this level's host [for recursive descent]
+ * @param level
+ * the level number
+ */
+ LoggedNetworkTopology(HashSet<ParsedHost> hosts, String name, int level) {
+
+ this.name = name;
+ this.children = null;
+
+ if (level < ParsedHost.numberOfDistances() - 1) {
+ HashMap<String, HashSet<ParsedHost>> topologies =
+ new HashMap<String, HashSet<ParsedHost>>();
+
+ Iterator<ParsedHost> iter = hosts.iterator();
+
+ while (iter.hasNext()) {
+ ParsedHost host = iter.next();
+
+ String thisComponent = host.nameComponent(level);
+
+ HashSet<ParsedHost> thisSet = topologies.get(thisComponent);
+
+ if (thisSet == null) {
+ thisSet = new HashSet<ParsedHost>();
+ topologies.put(thisComponent, thisSet);
+ }
+
+ thisSet.add(host);
+ }
+
+ children = new ArrayList<LoggedNetworkTopology>();
+
+ for (Map.Entry<String, HashSet<ParsedHost>> ent : topologies.entrySet()) {
+ children.add(new LoggedNetworkTopology(ent.getValue(), ent.getKey(),
+ level + 1));
+ }
+ } else {
+ // nothing to do here
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ void setName(String name) {
+ this.name = name;
+ }
+
+ public List<LoggedNetworkTopology> getChildren() {
+ return children;
+ }
+
+ void setChildren(List<LoggedNetworkTopology> children) {
+ this.children = children;
+ }
+
+ private void compare1(List<LoggedNetworkTopology> c1,
+ List<LoggedNetworkTopology> c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || c1.size() != c2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+
+ Collections.sort(c1, new TopoSort());
+ Collections.sort(c2, new TopoSort());
+
+ for (int i = 0; i < c1.size(); ++i) {
+ c1.get(i).deepCompare(c2.get(i), new TreePath(loc, eltname, i));
+ }
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedNetworkTopology)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedNetworkTopology other = (LoggedNetworkTopology) comparand;
+
+ compare1(children, other.children, loc, "children");
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
+/**
+ * A {@link LoggedSingleRelativeRanking} represents an X-Y coordinate of a
+ * single point in a discrete CDF.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedSingleRelativeRanking implements DeepCompare {
+ /**
+ * The Y coordinate, as a fraction {@code ( 0.0D, 1.0D )}. The default value
+ * is there to mark an unfilled-in value.
+ */
+ double relativeRanking = -1.0D;
+ /**
+ * The X coordinate
+ */
+ long datum = -1L;
+
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
+ public double getRelativeRanking() {
+ return relativeRanking;
+ }
+
+ void setRelativeRanking(double relativeRanking) {
+ this.relativeRanking = relativeRanking;
+ }
+
+ public long getDatum() {
+ return datum;
+ }
+
+ void setDatum(long datum) {
+ this.datum = datum;
+ }
+
+ private void compare1(long c1, long c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(double c1, double c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedSingleRelativeRanking)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedSingleRelativeRanking other = (LoggedSingleRelativeRanking) comparand;
+
+ compare1(relativeRanking, other.relativeRanking, loc, "relativeRanking");
+ compare1(datum, other.datum, loc, "datum");
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
+/**
+ * A {@link LoggedTask} represents a [hadoop] task that is part of a hadoop job.
+ * It knows about the [pssibly empty] sequence of attempts, its I/O footprint,
+ * and its runtime.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedTask implements DeepCompare {
+ long inputBytes = -1L;
+ long inputRecords = -1L;
+ long outputBytes = -1L;
+ long outputRecords = -1L;
+ String taskID;
+ long startTime = -1L;
+ long finishTime = -1L;
+ Pre21JobHistoryConstants.Values taskType;
+ Pre21JobHistoryConstants.Values taskStatus;
+ List<LoggedTaskAttempt> attempts = new ArrayList<LoggedTaskAttempt>();
+
+ ArrayList<LoggedLocation> preferredLocations =
+ new ArrayList<LoggedLocation>();
+
+ int numberMaps = -1;
+ int numberReduces = -1;
+
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
+ LoggedTask() {
+ super();
+ }
+
+ public long getInputBytes() {
+ return inputBytes;
+ }
+
+ void setInputBytes(long inputBytes) {
+ this.inputBytes = inputBytes;
+ }
+
+ public long getInputRecords() {
+ return inputRecords;
+ }
+
+ void setInputRecords(long inputRecords) {
+ this.inputRecords = inputRecords;
+ }
+
+ public long getOutputBytes() {
+ return outputBytes;
+ }
+
+ void setOutputBytes(long outputBytes) {
+ this.outputBytes = outputBytes;
+ }
+
+ public long getOutputRecords() {
+ return outputRecords;
+ }
+
+ void setOutputRecords(long outputRecords) {
+ this.outputRecords = outputRecords;
+ }
+
+ public String getTaskID() {
+ return taskID;
+ }
+
+ void setTaskID(String taskID) {
+ this.taskID = taskID;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ public List<LoggedTaskAttempt> getAttempts() {
+ return attempts;
+ }
+
+ void setAttempts(List<LoggedTaskAttempt> attempts) {
+ this.attempts = attempts;
+ }
+
+ public ArrayList<LoggedLocation> getPreferredLocations() {
+ return preferredLocations;
+ }
+
+ void setPreferredLocations(ArrayList<LoggedLocation> preferredLocations) {
+ this.preferredLocations = preferredLocations;
+ }
+
+ public int getNumberMaps() {
+ return numberMaps;
+ }
+
+ void setNumberMaps(int numberMaps) {
+ this.numberMaps = numberMaps;
+ }
+
+ public int getNumberReduces() {
+ return numberReduces;
+ }
+
+ void setNumberReduces(int numberReduces) {
+ this.numberReduces = numberReduces;
+ }
+
+ public Pre21JobHistoryConstants.Values getTaskStatus() {
+ return taskStatus;
+ }
+
+ void setTaskStatus(Pre21JobHistoryConstants.Values taskStatus) {
+ this.taskStatus = taskStatus;
+ }
+
+ public Pre21JobHistoryConstants.Values getTaskType() {
+ return taskType;
+ }
+
+ void setTaskType(Pre21JobHistoryConstants.Values taskType) {
+ this.taskType = taskType;
+ }
+
+ private void compare1(long c1, long c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(String c1, String c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+ if (c1 == null || c2 == null || !c1.equals(c2)) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(Pre21JobHistoryConstants.Values c1,
+ Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+ if (c1 == null || c2 == null || !c1.equals(c2)) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compareLoggedLocations(ArrayList<LoggedLocation> c1,
+ ArrayList<LoggedLocation> c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || c1.size() != c2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+
+ for (int i = 0; i < c1.size(); ++i) {
+ c1.get(i).deepCompare(c2.get(i), new TreePath(loc, eltname, i));
+ }
+ }
+
+ private void compareLoggedTaskAttempts(List<LoggedTaskAttempt> c1,
+ List<LoggedTaskAttempt> c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || c1.size() != c2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+
+ for (int i = 0; i < c1.size(); ++i) {
+ c1.get(i).deepCompare(c2.get(i), new TreePath(loc, eltname, i));
+ }
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedTask)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedTask other = (LoggedTask) comparand;
+
+ compare1(inputBytes, other.inputBytes, loc, "inputBytes");
+ compare1(inputRecords, other.inputRecords, loc, "inputRecords");
+ compare1(outputBytes, other.outputBytes, loc, "outputBytes");
+ compare1(outputRecords, other.outputRecords, loc, "outputRecords");
+
+ compare1(taskID, other.taskID, loc, "taskID");
+
+ compare1(startTime, other.startTime, loc, "startTime");
+ compare1(finishTime, other.finishTime, loc, "finishTime");
+
+ compare1(taskType, other.taskType, loc, "taskType");
+ compare1(taskStatus, other.taskStatus, loc, "taskStatus");
+
+ compareLoggedTaskAttempts(attempts, other.attempts, loc, "attempts");
+ compareLoggedLocations(preferredLocations, other.preferredLocations, loc,
+ "preferredLocations");
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools.rumen;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
+// HACK ALERT!!! This "should" have have two subclasses, which might be called
+// LoggedMapTaskAttempt and LoggedReduceTaskAttempt, but
+// the Jackson implementation of JSON doesn't handle a
+// superclass-valued field.
+
+/**
+ * A {@link LoggedTaskAttempt} represents an attempt to run an hadoop task in a
+ * hadoop job. Note that a task can have several attempts.
+ *
+ * All of the public methods are simply accessors for the instance variables we
+ * want to write out in the JSON files.
+ *
+ */
+public class LoggedTaskAttempt implements DeepCompare {
+
+ String attemptID;
+ Pre21JobHistoryConstants.Values result;
+ long startTime = -1L;
+ long finishTime = -1L;
+ String hostName;
+
+ long hdfsBytesRead = -1L;
+ long hdfsBytesWritten = -1L;
+ long fileBytesRead = -1L;
+ long fileBytesWritten = -1L;
+ long mapInputRecords = -1L;
+ long mapInputBytes = -1L;
+ long mapOutputBytes = -1L;
+ long mapOutputRecords = -1L;
+ long combineInputRecords = -1L;
+ long reduceInputGroups = -1L;
+ long reduceInputRecords = -1L;
+ long reduceShuffleBytes = -1L;
+ long reduceOutputRecords = -1L;
+ long spilledRecords = -1L;
+
+ long shuffleFinished = -1L;
+ long sortFinished = -1L;
+
+ LoggedLocation location;
+
+ LoggedTaskAttempt() {
+ super();
+ }
+
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
+ public long getShuffleFinished() {
+ return shuffleFinished;
+ }
+
+ void setShuffleFinished(long shuffleFinished) {
+ this.shuffleFinished = shuffleFinished;
+ }
+
+ public long getSortFinished() {
+ return sortFinished;
+ }
+
+ void setSortFinished(long sortFinished) {
+ this.sortFinished = sortFinished;
+ }
+
+ public String getAttemptID() {
+ return attemptID;
+ }
+
+ void setAttemptID(String attemptID) {
+ this.attemptID = attemptID;
+ }
+
+ public Pre21JobHistoryConstants.Values getResult() {
+ return result;
+ }
+
+ void setResult(Pre21JobHistoryConstants.Values result) {
+ this.result = result;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ public long getHdfsBytesRead() {
+ return hdfsBytesRead;
+ }
+
+ void setHdfsBytesRead(long hdfsBytesRead) {
+ this.hdfsBytesRead = hdfsBytesRead;
+ }
+
+ public long getHdfsBytesWritten() {
+ return hdfsBytesWritten;
+ }
+
+ void setHdfsBytesWritten(long hdfsBytesWritten) {
+ this.hdfsBytesWritten = hdfsBytesWritten;
+ }
+
+ public long getFileBytesRead() {
+ return fileBytesRead;
+ }
+
+ void setFileBytesRead(long fileBytesRead) {
+ this.fileBytesRead = fileBytesRead;
+ }
+
+ public long getFileBytesWritten() {
+ return fileBytesWritten;
+ }
+
+ void setFileBytesWritten(long fileBytesWritten) {
+ this.fileBytesWritten = fileBytesWritten;
+ }
+
+ public long getMapInputRecords() {
+ return mapInputRecords;
+ }
+
+ void setMapInputRecords(long mapInputRecords) {
+ this.mapInputRecords = mapInputRecords;
+ }
+
+ public long getMapOutputBytes() {
+ return mapOutputBytes;
+ }
+
+ void setMapOutputBytes(long mapOutputBytes) {
+ this.mapOutputBytes = mapOutputBytes;
+ }
+
+ public long getMapOutputRecords() {
+ return mapOutputRecords;
+ }
+
+ void setMapOutputRecords(long mapOutputRecords) {
+ this.mapOutputRecords = mapOutputRecords;
+ }
+
+ public long getCombineInputRecords() {
+ return combineInputRecords;
+ }
+
+ void setCombineInputRecords(long combineInputRecords) {
+ this.combineInputRecords = combineInputRecords;
+ }
+
+ public long getReduceInputGroups() {
+ return reduceInputGroups;
+ }
+
+ void setReduceInputGroups(long reduceInputGroups) {
+ this.reduceInputGroups = reduceInputGroups;
+ }
+
+ public long getReduceInputRecords() {
+ return reduceInputRecords;
+ }
+
+ void setReduceInputRecords(long reduceInputRecords) {
+ this.reduceInputRecords = reduceInputRecords;
+ }
+
+ public long getReduceShuffleBytes() {
+ return reduceShuffleBytes;
+ }
+
+ void setReduceShuffleBytes(long reduceShuffleBytes) {
+ this.reduceShuffleBytes = reduceShuffleBytes;
+ }
+
+ public long getReduceOutputRecords() {
+ return reduceOutputRecords;
+ }
+
+ void setReduceOutputRecords(long reduceOutputRecords) {
+ this.reduceOutputRecords = reduceOutputRecords;
+ }
+
+ public long getSpilledRecords() {
+ return spilledRecords;
+ }
+
+ void setSpilledRecords(long spilledRecords) {
+ this.spilledRecords = spilledRecords;
+ }
+
+ public LoggedLocation getLocation() {
+ return location;
+ }
+
+ void setLocation(LoggedLocation location) {
+ this.location = location;
+ }
+
+ public long getMapInputBytes() {
+ return mapInputBytes;
+ }
+
+ void setMapInputBytes(long mapInputBytes) {
+ this.mapInputBytes = mapInputBytes;
+ }
+
+ private void compare1(String c1, String c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ if (c1 == null || c2 == null || !c1.equals(c2)) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(long c1, long c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(Pre21JobHistoryConstants.Values c1,
+ Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (c1 != c2) {
+ throw new DeepInequalityException(eltname + " miscompared", new TreePath(
+ loc, eltname));
+ }
+ }
+
+ private void compare1(LoggedLocation c1, LoggedLocation c2, TreePath loc,
+ String eltname) throws DeepInequalityException {
+ if (c1 == null && c2 == null) {
+ return;
+ }
+
+ TreePath recurse = new TreePath(loc, eltname);
+
+ if (c1 == null || c2 == null) {
+ throw new DeepInequalityException(eltname + " miscompared", recurse);
+ }
+
+ c1.deepCompare(c2, recurse);
+ }
+
+ public void deepCompare(DeepCompare comparand, TreePath loc)
+ throws DeepInequalityException {
+ if (!(comparand instanceof LoggedTaskAttempt)) {
+ throw new DeepInequalityException("comparand has wrong type", loc);
+ }
+
+ LoggedTaskAttempt other = (LoggedTaskAttempt) comparand;
+
+ compare1(attemptID, other.attemptID, loc, "attemptID");
+ compare1(result, other.result, loc, "result");
+ compare1(startTime, other.startTime, loc, "startTime");
+ compare1(finishTime, other.finishTime, loc, "finishTime");
+ compare1(hostName, other.hostName, loc, "hostName");
+
+ compare1(hdfsBytesRead, other.hdfsBytesRead, loc, "hdfsBytesRead");
+ compare1(hdfsBytesWritten, other.hdfsBytesWritten, loc, "hdfsBytesWritten");
+ compare1(fileBytesRead, other.fileBytesRead, loc, "fileBytesRead");
+ compare1(fileBytesWritten, other.fileBytesWritten, loc, "fileBytesWritten");
+ compare1(mapInputBytes, other.mapInputBytes, loc, "mapInputBytes");
+ compare1(mapInputRecords, other.mapInputRecords, loc, "mapInputRecords");
+ compare1(mapOutputBytes, other.mapOutputBytes, loc, "mapOutputBytes");
+ compare1(mapOutputRecords, other.mapOutputRecords, loc, "mapOutputRecords");
+ compare1(combineInputRecords, other.combineInputRecords, loc,
+ "combineInputRecords");
+ compare1(reduceInputGroups, other.reduceInputGroups, loc,
+ "reduceInputGroups");
+ compare1(reduceInputRecords, other.reduceInputRecords, loc,
+ "reduceInputRecords");
+ compare1(reduceShuffleBytes, other.reduceShuffleBytes, loc,
+ "reduceShuffleBytes");
+ compare1(reduceOutputRecords, other.reduceOutputRecords, loc,
+ "reduceOutputRecords");
+ compare1(spilledRecords, other.spilledRecords, loc, "spilledRecords");
+
+ compare1(shuffleFinished, other.shuffleFinished, loc, "shuffleFinished");
+ compare1(sortFinished, other.sortFinished, loc, "sortFinished");
+
+ compare1(location, other.location, loc, "location");
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MachineNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MachineNode.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MachineNode.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MachineNode.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+/**
+ * {@link MachineNode} represents the configuration of a cluster node.
+ * {@link MachineNode} should be constructed by {@link MachineNode.Builder}.
+ */
+public final class MachineNode extends Node {
+ long memory = -1; // in KB
+ int mapSlots = 1;
+ int reduceSlots = 1;
+ long memoryPerMapSlot = -1; // in KB
+ long memoryPerReduceSlot = -1; // in KB
+ int numCores = 1;
+
+ MachineNode(String name, int level) {
+ super(name, level);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // name/level sufficient
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ // match equals
+ return super.hashCode();
+ }
+
+ /**
+ * Get the available physical RAM of the node.
+ * @return The available physical RAM of the node, in KB.
+ */
+ public long getMemory() {
+ return memory;
+ }
+
+ /**
+ * Get the number of map slots of the node.
+ * @return The number of map slots of the node.
+ */
+ public int getMapSlots() {
+ return mapSlots;
+ }
+
+ /**
+ * Get the number of reduce slots of the node.
+ * @return The number of reduce slots fo the node.
+ */
+ public int getReduceSlots() {
+ return reduceSlots;
+ }
+
+ /**
+ * Get the amount of RAM reserved for each map slot.
+ * @return the amount of RAM reserved for each map slot, in KB.
+ */
+ public long getMemoryPerMapSlot() {
+ return memoryPerMapSlot;
+ }
+
+ /**
+ * Get the amount of RAM reserved for each reduce slot.
+ * @return the amount of RAM reserved for each reduce slot, in KB.
+ */
+ public long getMemoryPerReduceSlot() {
+ return memoryPerReduceSlot;
+ }
+
+ /**
+ * Get the number of cores of the node.
+ * @return the number of cores of the node.
+ */
+ public int getNumCores() {
+ return numCores;
+ }
+
+ /**
+ * Get the rack node that the machine belongs to.
+ *
+ * @return The rack node that the machine belongs to. Returns null if the
+ * machine does not belong to any rack.
+ */
+ public RackNode getRackNode() {
+ return (RackNode)getParent();
+ }
+
+ @Override
+ public synchronized boolean addChild(Node child) {
+ throw new IllegalStateException("Cannot add child to MachineNode");
+ }
+
+ /**
+ * Builder for a NodeInfo object
+ */
+ public static final class Builder {
+ private MachineNode node;
+
+ /**
+ * Start building a new NodeInfo object.
+ * @param name
+ * Unique name of the node. Typically the fully qualified domain
+ * name.
+ */
+ public Builder(String name, int level) {
+ node = new MachineNode(name, level);
+ }
+
+ /**
+ * Set the physical memory of the node.
+ * @param memory Available RAM in KB.
+ */
+ public Builder setMemory(long memory) {
+ node.memory = memory;
+ return this;
+ }
+
+ /**
+ * Set the number of map slot for the node.
+ * @param mapSlots The number of map slots for the node.
+ */
+ public Builder setMapSlots(int mapSlots) {
+ node.mapSlots = mapSlots;
+ return this;
+ }
+
+ /**
+ * Set the number of reduce slot for the node.
+ * @param reduceSlots The number of reduce slots for the node.
+ */
+ public Builder setReduceSlots(int reduceSlots) {
+ node.reduceSlots = reduceSlots;
+ return this;
+ }
+
+ /**
+ * Set the amount of RAM reserved for each map slot.
+ * @param memoryPerMapSlot The amount of RAM reserved for each map slot, in KB.
+ */
+ public Builder setMemoryPerMapSlot(long memoryPerMapSlot) {
+ node.memoryPerMapSlot = memoryPerMapSlot;
+ return this;
+ }
+
+ /**
+ * Set the amount of RAM reserved for each reduce slot.
+ * @param memoryPerReduceSlot The amount of RAM reserved for each reduce slot, in KB.
+ */
+ public Builder setMemoryPerReduceSlot(long memoryPerReduceSlot) {
+ node.memoryPerReduceSlot = memoryPerReduceSlot;
+ return this;
+ }
+
+ /**
+ * Set the number of cores for the node.
+ * @param numCores Number of cores for the node.
+ */
+ public Builder setNumCores(int numCores) {
+ node.numCores = numCores;
+ return this;
+ }
+
+ /**
+ * Clone the settings from a reference {@link MachineNode} object.
+ * @param ref The reference {@link MachineNode} object.
+ */
+ public Builder cloneFrom(MachineNode ref) {
+ node.memory = ref.memory;
+ node.mapSlots = ref.mapSlots;
+ node.reduceSlots = ref.reduceSlots;
+ node.memoryPerMapSlot = ref.memoryPerMapSlot;
+ node.memoryPerReduceSlot = ref.memoryPerReduceSlot;
+ node.numCores = ref.numCores;
+ return this;
+ }
+
+ /**
+ * Build the {@link MachineNode} object.
+ * @return The {@link MachineNode} object being built.
+ */
+ public MachineNode build() {
+ MachineNode retVal = node;
+ node = null;
+ return retVal;
+ }
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java?rev=1077079&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/tools/org/apache/hadoop/tools/rumen/MapTaskAttemptInfo.java Fri Mar 4 03:38:20 2011
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.rumen;
+
+import org.apache.hadoop.mapred.TaskStatus.State;
+
+/**
+ * {@link MapTaskAttemptInfo} represents the information with regard to a
+ * map task attempt.
+ */
+public class MapTaskAttemptInfo extends TaskAttemptInfo {
+ private long runtime;
+
+ public MapTaskAttemptInfo(State state, TaskInfo taskInfo, long runtime) {
+ super(state, taskInfo);
+ this.runtime = runtime;
+ }
+
+ @Override
+ public long getRuntime() {
+ return getMapRuntime();
+ }
+
+ /**
+ * Get the runtime for the <b>map</b> phase of the map-task attempt.
+ *
+ * @return the runtime for the <b>map</b> phase of the map-task attempt
+ */
+ public long getMapRuntime() {
+ return runtime;
+ }
+
+}