You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/10/16 08:06:21 UTC
svn commit: r1398658 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/metrics2/lib/
src/main/java/org/apache/hadoop/metrics2/util/
src/test/java/org/apache/hadoop/metrics2/util/
Author: todd
Date: Tue Oct 16 06:06:20 2012
New Revision: 1398658
URL: http://svn.apache.org/viewvc?rev=1398658&view=rev
Log:
HADOOP-8929. Add toString, other improvements for SampleQuantiles. Contributed by Todd Lipcon.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1398658&r1=1398657&r2=1398658&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct 16 06:06:20 2012
@@ -318,6 +318,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8784. Improve IPC.Client's token use (daryn)
+ HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java?rev=1398658&r1=1398657&r2=1398658&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java Tue Oct 16 06:06:20 2012
@@ -20,7 +20,6 @@ package org.apache.hadoop.metrics2.lib;
import static org.apache.hadoop.metrics2.lib.Interns.info;
-import java.io.IOException;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -150,14 +149,8 @@ public class MutableQuantiles extends Mu
@Override
public void run() {
synchronized (parent) {
- try {
- parent.previousCount = parent.estimator.getCount();
- parent.previousSnapshot = parent.estimator.snapshot();
- } catch (IOException e) {
- // Couldn't get a new snapshot because the window was empty
- parent.previousCount = 0;
- parent.previousSnapshot = null;
- }
+ parent.previousCount = parent.estimator.getCount();
+ parent.previousSnapshot = parent.estimator.snapshot();
parent.estimator.clear();
}
parent.setChanged();
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java?rev=1398658&r1=1398657&r2=1398658&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java Tue Oct 16 06:06:20 2012
@@ -20,12 +20,14 @@ package org.apache.hadoop.metrics2.util;
import org.apache.hadoop.classification.InterfaceAudience;
+import com.google.common.collect.ComparisonChain;
+
/**
* Specifies a quantile (with error bounds) to be watched by a
* {@link SampleQuantiles} object.
*/
@InterfaceAudience.Private
-public class Quantile {
+public class Quantile implements Comparable<Quantile> {
public final double quantile;
public final double error;
@@ -57,4 +59,19 @@ public class Quantile {
return (int) (Double.doubleToLongBits(quantile) ^ Double
.doubleToLongBits(error));
}
+
+ @Override
+ public int compareTo(Quantile other) {
+ return ComparisonChain.start()
+ .compare(quantile, other.quantile)
+ .compare(error, other.error)
+ .result();
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%.2f %%ile +/- %.2f%%",
+ quantile * 100, error * 100);
+ }
+
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java?rev=1398658&r1=1398657&r2=1398658&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java Tue Oct 16 06:06:20 2012
@@ -18,16 +18,17 @@
package org.apache.hadoop.metrics2.util;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.classification.InterfaceAudience;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
/**
* Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm
@@ -202,10 +203,8 @@ public class SampleQuantiles {
* @param quantile Queried quantile, e.g. 0.50 or 0.99.
* @return Estimated value at that quantile.
*/
- private long query(double quantile) throws IOException {
- if (samples.size() == 0) {
- throw new IOException("No samples present");
- }
+ private long query(double quantile) {
+ Preconditions.checkState(!samples.isEmpty(), "no data in estimator");
int rankMin = 0;
int desired = (int) (quantile * count);
@@ -231,14 +230,18 @@ public class SampleQuantiles {
/**
* Get a snapshot of the current values of all the tracked quantiles.
*
- * @return snapshot of the tracked quantiles
- * @throws IOException
- * if no items have been added to the estimator
+ * @return snapshot of the tracked quantiles. If no items are added
+ * to the estimator, returns null.
*/
- synchronized public Map<Quantile, Long> snapshot() throws IOException {
+ synchronized public Map<Quantile, Long> snapshot() {
// flush the buffer first for best results
insertBatch();
- Map<Quantile, Long> values = new HashMap<Quantile, Long>(quantiles.length);
+
+ if (samples.isEmpty()) {
+ return null;
+ }
+
+ Map<Quantile, Long> values = new TreeMap<Quantile, Long>();
for (int i = 0; i < quantiles.length; i++) {
values.put(quantiles[i], query(quantiles[i].quantile));
}
@@ -273,6 +276,16 @@ public class SampleQuantiles {
bufferCount = 0;
samples.clear();
}
+
+ @Override
+ synchronized public String toString() {
+ Map<Quantile, Long> data = snapshot();
+ if (data == null) {
+ return "[no samples]";
+ } else {
+ return Joiner.on("\n").withKeyValueSeparator(": ").join(data);
+ }
+ }
/**
* Describes a measured value passed to the estimator, tracking additional
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java?rev=1398658&r1=1398657&r2=1398658&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java Tue Oct 16 06:06:20 2012
@@ -18,9 +18,7 @@
package org.apache.hadoop.metrics2.util;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
@@ -28,7 +26,6 @@ import java.util.Collections;
import java.util.Map;
import java.util.Random;
-import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
@@ -54,18 +51,22 @@ public class TestSampleQuantiles {
// Counts start off zero
assertEquals(estimator.getCount(), 0);
assertEquals(estimator.getSampleCount(), 0);
- try {
- estimator.snapshot();
- fail("Expected IOException from empty window");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("No samples", e);
- }
+
+ // Snapshot should be null if there are no entries.
+ assertNull(estimator.snapshot());
// Count increment correctly by 1
estimator.insert(1337);
assertEquals(estimator.getCount(), 1);
estimator.snapshot();
assertEquals(estimator.getSampleCount(), 1);
+
+ assertEquals(
+ "50.00 %ile +/- 5.00%: 1337\n" +
+ "75.00 %ile +/- 2.50%: 1337\n" +
+ "90.00 %ile +/- 1.00%: 1337\n" +
+ "95.00 %ile +/- 0.50%: 1337\n" +
+ "99.00 %ile +/- 0.10%: 1337", estimator.toString());
}
/**
@@ -80,12 +81,7 @@ public class TestSampleQuantiles {
estimator.clear();
assertEquals(estimator.getCount(), 0);
assertEquals(estimator.getSampleCount(), 0);
- try {
- estimator.snapshot();
- fail("Expected IOException for an empty window.");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("No samples", e);
- }
+ assertNull(estimator.snapshot());
}
/**