You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ss...@apache.org on 2012/10/19 20:49:42 UTC
svn commit: r1400219 - in
/hadoop/common/branches/MR-3902/hadoop-common-project:
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/
hadoop-common/ hado...
Author: sseth
Date: Fri Oct 19 18:49:38 2012
New Revision: 1400219
URL: http://svn.apache.org/viewvc?rev=1400219&view=rev
Log:
merge from trunk to branch MR-3902
Removed:
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/packages/
Modified:
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java
hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java Fri Oct 19 18:49:38 2012
@@ -158,7 +158,7 @@ public class KerberosAuthenticator imple
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.connect();
- if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
+ if (conn.getRequestProperty(AUTHORIZATION) != null && conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
LOG.debug("JDK performed authentication on our behalf.");
// If the JDK already did the SPNEGO back-and-forth for
// us, just pull out the token.
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java Fri Oct 19 18:49:38 2012
@@ -134,9 +134,11 @@ public abstract class AuthenticatorTestC
try {
URL url = new URL(getBaseURL());
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ Assert.assertFalse(token.isSet());
TestConnectionConfigurator connConf = new TestConnectionConfigurator();
AuthenticatedURL aUrl = new AuthenticatedURL(authenticator, connConf);
HttpURLConnection conn = aUrl.openConnection(url, token);
+ Assert.assertTrue(token.isSet());
Assert.assertTrue(connConf.invoked);
String tokenStr = token.toString();
if (doPost) {
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java Fri Oct 19 18:49:38 2012
@@ -43,6 +43,14 @@ public class TestKerberosAuthenticator e
_testAuthentication(new KerberosAuthenticator(), false);
}
+ public void testFallbacktoPseudoAuthenticatorAnonymous() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(AuthenticationFilter.AUTH_TYPE, "simple");
+ props.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "true");
+ setAuthenticationHandlerConfig(props);
+ _testAuthentication(new KerberosAuthenticator(), false);
+ }
+
public void testNotAuthenticated() throws Exception {
setAuthenticationHandlerConfig(getAuthenticationHandlerConfiguration());
start();
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct 19 18:49:38 2012
@@ -72,8 +72,8 @@ Trunk (Unreleased)
HADOOP-8360. empty-configuration.xml fails xml validation
(Radim Kolar via harsh)
- HADOOP-8367 Improve documentation of declaringClassProtocolName in rpc headers
- (Sanjay Radia)
+ HADOOP-8367 Improve documentation of declaringClassProtocolName in
+ rpc headers. (Sanjay Radia)
HADOOP-8415. Add getDouble() and setDouble() in
org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
@@ -124,6 +124,11 @@ Trunk (Unreleased)
HADOOP-8910. Add examples to GlobExpander#expand method. (suresh)
+ HADOOP-8920. Add more javadoc to metrics2 related classes. (suresh)
+
+ HADOOP-8776. Provide an option in test-patch that can enable/disable
+ compiling native code. (Chris Nauroth via suresh)
+
BUG FIXES
HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
@@ -316,11 +321,23 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8784. Improve IPC.Client's token use (daryn)
+ HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
+
+ HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
+ javascript in browser dashboard (Damien Hardy via bobby)
+
+ HADOOP-8931. Add Java version to startup message. (eli)
+
+ HADOOP-8925. Remove the packaging. (eli)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
via atm)
+ HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
+ data (Gopal V via bobby)
+
BUG FIXES
HADOOP-8795. BASH tab completion doesn't look in PATH, assumes path to
@@ -351,10 +368,15 @@ Release 2.0.3-alpha - Unreleased
webhdfs filesystem and fsck to fail when security is on.
(Arpit Gupta via suresh)
-
HADOOP-8901. GZip and Snappy support may not work without unversioned
libraries (Colin Patrick McCabe via todd)
+ HADOOP-8883. Anonymous fallback in KerberosAuthenticator is broken.
+ (rkanter via tucu)
+
+ HADOOP-8900. BuiltInGzipDecompressor throws IOException - stored gzip size
+ doesn't match decompressed size. (Slavik Krassovsky via suresh)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -1053,6 +1075,12 @@ Release 0.23.5 - UNRELEASED
IMPROVEMENTS
+ HADOOP-8923. JNI-based user-group mapping modules can be too chatty on
+ lookup failures. (Kihwal Lee via suresh)
+
+ HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via
+ bobby)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java Fri Oct 19 18:49:38 2012
@@ -387,7 +387,7 @@ public class BuiltInGzipDecompressor imp
copyBytesToLocal(n); // modifies userBufLen, etc.
if (localBufOff >= 4) { // should be strictly ==
long inputSize = readUIntLE(localBuf, 0);
- if (inputSize != (inflater.getBytesWritten() & 0xffffffff)) {
+ if (inputSize != (inflater.getBytesWritten() & 0xffffffffL)) {
throw new IOException(
"stored gzip size doesn't match decompressed size");
}
@@ -571,7 +571,7 @@ public class BuiltInGzipDecompressor imp
return ((((long)(b[off+3] & 0xff) << 24) |
((long)(b[off+2] & 0xff) << 16) |
((long)(b[off+1] & 0xff) << 8) |
- ((long)(b[off] & 0xff) )) & 0xffffffff);
+ ((long)(b[off] & 0xff) )) & 0xffffffffL);
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java Fri Oct 19 18:49:38 2012
@@ -113,12 +113,17 @@ import org.codehaus.jackson.JsonGenerato
* All other objects will be converted to a string and output as such.
*
* The bean's name and modelerType will be returned for all beans.
+ *
+ * Optional paramater "callback" should be used to deliver JSONP response.
+ *
*/
public class JMXJsonServlet extends HttpServlet {
private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
private static final long serialVersionUID = 1L;
+ private static final String CALLBACK_PARAM = "callback";
+
/**
* MBean server.
*/
@@ -154,11 +159,22 @@ public class JMXJsonServlet extends Http
return;
}
JsonGenerator jg = null;
+ String jsonpcb = null;
+ PrintWriter writer = null;
try {
- response.setContentType("application/json; charset=utf8");
+ writer = response.getWriter();
+
+ // "callback" parameter implies JSONP outpout
+ jsonpcb = request.getParameter(CALLBACK_PARAM);
+ if (jsonpcb != null) {
+ response.setContentType("application/javascript; charset=utf8");
+ writer.write(jsonpcb + "(");
+ } else {
+ response.setContentType("application/json; charset=utf8");
+ }
- PrintWriter writer = response.getWriter();
jg = jsonFactory.createJsonGenerator(writer);
+ jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
jg.useDefaultPrettyPrinter();
jg.writeStartObject();
@@ -188,6 +204,12 @@ public class JMXJsonServlet extends Http
if (jg != null) {
jg.close();
}
+ if (jsonpcb != null) {
+ writer.write(");");
+ }
+ if (writer != null) {
+ writer.close();
+ }
}
} catch (IOException e) {
LOG.error("Caught an exception while processing JMX request", e);
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java Fri Oct 19 18:49:38 2012
@@ -30,14 +30,14 @@ public interface MetricsCollector {
/**
* Add a metrics record
* @param name of the record
- * @return a metrics record builder for the record
+ * @return a {@link MetricsRecordBuilder} for the record {@code name}
*/
public MetricsRecordBuilder addRecord(String name);
/**
* Add a metrics record
* @param info of the record
- * @return a metrics record builder for the record
+ * @return a {@link MetricsRecordBuilder} for metrics {@code info}
*/
public MetricsRecordBuilder addRecord(MetricsInfo info);
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java Fri Oct 19 18:49:38 2012
@@ -22,7 +22,9 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
/**
- * The metrics filter interface
+ * The metrics filter interface. The MetricsFilter objects can be used either to
+ * filter the metrics from {@link MetricsSource}s or to filter metrics per
+ * {@link MetricsSink}.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java Fri Oct 19 18:49:38 2012
@@ -20,19 +20,24 @@ package org.apache.hadoop.metrics2;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.annotation.Metric;
/**
- * Interface to provide immutable meta info for metrics
+ * Interface to provide immutable metainfo for metrics.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface MetricsInfo {
/**
+ * Typically name corresponds to annotation {@link Metric#value()} or
+ * the name of the class.
* @return the name of the metric/tag
*/
String name();
/**
+ * Typically the description corresponds to annotation {@link Metric#about()}
+ * or the name of the class.
* @return the description of the metric/tag
*/
String description();
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java Fri Oct 19 18:49:38 2012
@@ -36,17 +36,17 @@ public interface MetricsRecord {
long timestamp();
/**
- * @return the record name
+ * @return the metrics record name
*/
String name();
/**
- * @return the description of the record
+ * @return the description of the metrics record
*/
String description();
/**
- * @return the context name of the record
+ * @return the context name of the metrics record
*/
String context();
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java Fri Oct 19 18:49:38 2012
@@ -28,7 +28,7 @@ import org.apache.hadoop.classification.
@InterfaceStability.Evolving
public abstract class MetricsRecordBuilder {
/**
- * Add a metrics tag
+ * Add a metrics value with metrics information
* @param info metadata of the tag
* @param value of the tag
* @return self
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java Fri Oct 19 18:49:38 2012
@@ -22,7 +22,11 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
/**
- * The metrics sink interface
+ * The metrics sink interface. <p>
+ * Implementations of this interface consume the {@link MetricsRecord} generated
+ * from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ * periodically pushes the {@link MetricsRecord} to the sink using
+ * {@link #putMetrics(MetricsRecord)} method.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java Fri Oct 19 18:49:38 2012
@@ -22,13 +22,15 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
/**
- * The metrics source interface
+ * The source of metrics information. It generates and updates metrics. It
+ * registers with {@link MetricsSystem}, which periodically polls it to collect
+ * {@link MetricsRecord} and passes it to {@link MetricsSink}.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface MetricsSource {
/**
- * Get metrics from the source
+ * Get metrics from the metrics source
* @param collector to contain the resulting metrics snapshot
* @param all if true, return all metrics even if unchanged.
*/
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java Fri Oct 19 18:49:38 2012
@@ -22,7 +22,18 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
/**
- * The metrics system interface
+ * The metrics system interface.
+ *
+ * The following components are used for metrics.
+ * <ul>
+ * <li>{@link MetricsSource} generate and update metrics information.</li>
+ * <li>{@link MetricsSink} consume the metrics information</li>
+ * </ul>
+ *
+ * {@link MetricsSource} and {@link MetricsSink} register with the metrics
+ * system. Implementations of {@link MetricsSystem} polls the
+ * {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to
+ * {@link MetricsSink}.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java Fri Oct 19 18:49:38 2012
@@ -24,7 +24,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
/**
- * Annotation interface for a single metric
+ * Annotation interface for a single metric used to annotate a field or a method
+ * in the class.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java Fri Oct 19 18:49:38 2012
@@ -32,6 +32,14 @@ import org.apache.hadoop.metrics2.Metric
import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.util.Time;
+/**
+ * {@link MetricsRecordBuilder} implementation used for building metrics records
+ * by the {@link MetricsCollector}. It provides the following functionality:
+ * <ul>
+ * <li>Allows configuring filters for metrics.
+ * </ul>
+ *
+ */
class MetricsRecordBuilderImpl extends MetricsRecordBuilder {
private final MetricsCollector parent;
private final long timestamp;
@@ -41,9 +49,15 @@ class MetricsRecordBuilderImpl extends M
private final MetricsFilter recordFilter, metricFilter;
private final boolean acceptable;
+ /**
+ * @param parent {@link MetricsCollector} using this record builder
+ * @param info metrics information
+ * @param rf
+ * @param mf
+ * @param acceptable
+ */
MetricsRecordBuilderImpl(MetricsCollector parent, MetricsInfo info,
- MetricsFilter rf, MetricsFilter mf,
- boolean acceptable) {
+ MetricsFilter rf, MetricsFilter mf, boolean acceptable) {
this.parent = parent;
timestamp = Time.now();
recInfo = info;
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java Fri Oct 19 18:49:38 2012
@@ -37,7 +37,7 @@ class MetricsRecordImpl extends Abstract
/**
* Construct a metrics record
- * @param info {@link MetricInfo} of the record
+ * @param info {@link MetricsInfo} of the record
* @param timestamp of the record
* @param tags of the record
* @param metrics of the record
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java Fri Oct 19 18:49:38 2012
@@ -235,11 +235,9 @@ public class MetricsSystemImpl extends M
void registerSource(String name, String desc, MetricsSource source) {
checkNotNull(config, "config");
MetricsConfig conf = sourceConfigs.get(name);
- MetricsSourceAdapter sa = conf != null
- ? new MetricsSourceAdapter(prefix, name, desc, source,
- injectedTags, period, conf)
- : new MetricsSourceAdapter(prefix, name, desc, source,
- injectedTags, period, config.subset(SOURCE_KEY));
+ MetricsSourceAdapter sa = new MetricsSourceAdapter(prefix, name, desc,
+ source, injectedTags, period, conf != null ? conf
+ : config.subset(SOURCE_KEY));
sources.put(name, sa);
sa.start();
LOG.debug("Registered source "+ name);
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java Fri Oct 19 18:49:38 2012
@@ -27,8 +27,13 @@ import org.apache.hadoop.metrics2.Metric
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
+import com.google.common.annotations.VisibleForTesting;
+
/**
- * The default metrics system singleton
+ * The default metrics system singleton. This class is used by all the daemon
+ * processes(such as NameNode, DataNode, JobTracker etc.). During daemon process
+ * initialization the processes call {@link DefaultMetricsSystem#init(String)}
+ * to initialize the {@link MetricsSystem}.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -37,7 +42,10 @@ public enum DefaultMetricsSystem {
private AtomicReference<MetricsSystem> impl =
new AtomicReference<MetricsSystem>(new MetricsSystemImpl());
+
+ @VisibleForTesting
volatile boolean miniClusterMode = false;
+
final UniqueNames mBeanNames = new UniqueNames();
final UniqueNames sourceNames = new UniqueNames();
@@ -87,12 +95,12 @@ public enum DefaultMetricsSystem {
MetricsSystem getImpl() { return impl.get(); }
- @InterfaceAudience.Private
+ @VisibleForTesting
public static void setMiniClusterMode(boolean choice) {
INSTANCE.miniClusterMode = choice;
}
- @InterfaceAudience.Private
+ @VisibleForTesting
public static boolean inMiniClusterMode() {
return INSTANCE.miniClusterMode;
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java Fri Oct 19 18:49:38 2012
@@ -35,7 +35,16 @@ import org.apache.hadoop.metrics2.annota
import org.apache.hadoop.metrics2.annotation.Metrics;
/**
- * Helper class to build metrics source object from annotations
+ * Helper class to build {@link MetricsSource} object from annotations.
+ * <p>
+ * For a given source object:
+ * <ul>
+ * <li>Sets the {@link Field}s annotated with {@link Metric} to
+ * {@link MutableMetric} and adds it to the {@link MetricsRegistry}.</li>
+ * <li>
+ * For {@link Method}s annotated with {@link Metric} creates
+ * {@link MutableMetric} and adds it to the {@link MetricsRegistry}.</li>
+ * </ul>
*/
@InterfaceAudience.Private
public class MetricsSourceBuilder {
@@ -115,9 +124,15 @@ public class MetricsSourceBuilder {
return r;
}
+ /**
+ * Change the declared field {@code field} in {@code source} Object to
+ * {@link MutableMetric}
+ */
private void add(Object source, Field field) {
for (Annotation annotation : field.getAnnotations()) {
- if (!(annotation instanceof Metric)) continue;
+ if (!(annotation instanceof Metric)) {
+ continue;
+ }
try {
// skip fields already set
field.setAccessible(true);
@@ -131,7 +146,7 @@ public class MetricsSourceBuilder {
registry);
if (mutable != null) {
try {
- field.set(source, mutable);
+ field.set(source, mutable); // Set the source field to MutableMetric
hasAtMetric = true;
} catch (Exception e) {
throw new MetricsException("Error setting field "+ field +
@@ -141,9 +156,12 @@ public class MetricsSourceBuilder {
}
}
+ /** Add {@link MutableMetric} for a method annotated with {@link Metric} */
private void add(Object source, Method method) {
for (Annotation annotation : method.getAnnotations()) {
- if (!(annotation instanceof Metric)) continue;
+ if (!(annotation instanceof Metric)) {
+ continue;
+ }
factory.newForMethod(source, method, (Metric) annotation, registry);
hasAtMetric = true;
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java Fri Oct 19 18:49:38 2012
@@ -130,6 +130,10 @@ public class MutableMetricsFactory {
return Interns.info(name2, about.isEmpty() ? name2 : about);
}
+ /**
+ * Remove the prefix "get", if any, from the method name. Return the
+ * capacitalized method name."
+ */
protected String getName(Method method) {
String methodName = method.getName();
if (methodName.startsWith("get")) {
@@ -140,12 +144,15 @@ public class MutableMetricsFactory {
protected MetricsInfo getInfo(Metric annotation, String defaultName) {
String[] value = annotation.value();
- if (value.length == 2) {
+ if (value.length == 2) {
+ // Use name and description from the annotation
return Interns.info(value[0], value[1]);
}
if (value.length == 1) {
+ // Use description from the annotation and method name as metric name
return Interns.info(defaultName, value[0]);
}
+ // Use method name as metric name and description
return Interns.info(defaultName, defaultName);
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java Fri Oct 19 18:49:38 2012
@@ -20,7 +20,6 @@ package org.apache.hadoop.metrics2.lib;
import static org.apache.hadoop.metrics2.lib.Interns.info;
-import java.io.IOException;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -150,14 +149,8 @@ public class MutableQuantiles extends Mu
@Override
public void run() {
synchronized (parent) {
- try {
- parent.previousCount = parent.estimator.getCount();
- parent.previousSnapshot = parent.estimator.snapshot();
- } catch (IOException e) {
- // Couldn't get a new snapshot because the window was empty
- parent.previousCount = 0;
- parent.previousSnapshot = null;
- }
+ parent.previousCount = parent.estimator.getCount();
+ parent.previousSnapshot = parent.estimator.snapshot();
parent.estimator.clear();
}
parent.setChanged();
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java Fri Oct 19 18:49:38 2012
@@ -20,12 +20,14 @@ package org.apache.hadoop.metrics2.util;
import org.apache.hadoop.classification.InterfaceAudience;
+import com.google.common.collect.ComparisonChain;
+
/**
* Specifies a quantile (with error bounds) to be watched by a
* {@link SampleQuantiles} object.
*/
@InterfaceAudience.Private
-public class Quantile {
+public class Quantile implements Comparable<Quantile> {
public final double quantile;
public final double error;
@@ -57,4 +59,19 @@ public class Quantile {
return (int) (Double.doubleToLongBits(quantile) ^ Double
.doubleToLongBits(error));
}
+
+ @Override
+ public int compareTo(Quantile other) {
+ return ComparisonChain.start()
+ .compare(quantile, other.quantile)
+ .compare(error, other.error)
+ .result();
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%.2f %%ile +/- %.2f%%",
+ quantile * 100, error * 100);
+ }
+
}
\ No newline at end of file
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java Fri Oct 19 18:49:38 2012
@@ -18,16 +18,17 @@
package org.apache.hadoop.metrics2.util;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.classification.InterfaceAudience;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
/**
* Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm
@@ -202,10 +203,8 @@ public class SampleQuantiles {
* @param quantile Queried quantile, e.g. 0.50 or 0.99.
* @return Estimated value at that quantile.
*/
- private long query(double quantile) throws IOException {
- if (samples.size() == 0) {
- throw new IOException("No samples present");
- }
+ private long query(double quantile) {
+ Preconditions.checkState(!samples.isEmpty(), "no data in estimator");
int rankMin = 0;
int desired = (int) (quantile * count);
@@ -231,14 +230,18 @@ public class SampleQuantiles {
/**
* Get a snapshot of the current values of all the tracked quantiles.
*
- * @return snapshot of the tracked quantiles
- * @throws IOException
- * if no items have been added to the estimator
+ * @return snapshot of the tracked quantiles. If no items are added
+ * to the estimator, returns null.
*/
- synchronized public Map<Quantile, Long> snapshot() throws IOException {
+ synchronized public Map<Quantile, Long> snapshot() {
// flush the buffer first for best results
insertBatch();
- Map<Quantile, Long> values = new HashMap<Quantile, Long>(quantiles.length);
+
+ if (samples.isEmpty()) {
+ return null;
+ }
+
+ Map<Quantile, Long> values = new TreeMap<Quantile, Long>();
for (int i = 0; i < quantiles.length; i++) {
values.put(quantiles[i], query(quantiles[i].quantile));
}
@@ -273,6 +276,16 @@ public class SampleQuantiles {
bufferCount = 0;
samples.clear();
}
+
+ @Override
+ synchronized public String toString() {
+ Map<Quantile, Long> data = snapshot();
+ if (data == null) {
+ return "[no samples]";
+ } else {
+ return Joiner.on("\n").withKeyValueSeparator(": ").join(data);
+ }
+ }
/**
* Describes a measured value passed to the estimator, tracking additional
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java Fri Oct 19 18:49:38 2012
@@ -48,7 +48,7 @@ public class JniBasedUnixGroupsMapping i
throw new RuntimeException("Bailing out since native library couldn't " +
"be loaded");
}
- LOG.info("Using JniBasedUnixGroupsMapping for Group resolution");
+ LOG.debug("Using JniBasedUnixGroupsMapping for Group resolution");
}
@Override
@@ -57,7 +57,11 @@ public class JniBasedUnixGroupsMapping i
try {
groups = getGroupForUser(user);
} catch (Exception e) {
- LOG.warn("Error getting groups for " + user, e);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Error getting groups for " + user, e);
+ } else {
+ LOG.info("Error getting groups for " + user + ": " + e.getMessage());
+ }
}
return Arrays.asList(groups);
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java Fri Oct 19 18:49:38 2012
@@ -52,7 +52,7 @@ public class JniBasedUnixGroupsNetgroupM
throw new RuntimeException("Bailing out since native library couldn't " +
"be loaded");
}
- LOG.info("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
+ LOG.debug("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
}
/**
@@ -115,7 +115,12 @@ public class JniBasedUnixGroupsNetgroupM
// JNI code does not expect '@' at the begining of the group name
users = getUsersForNetgroupJNI(netgroup.substring(1));
} catch (Exception e) {
- LOG.warn("error getting users for netgroup " + netgroup, e);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Error getting users for netgroup " + netgroup, e);
+ } else {
+ LOG.info("Error getting users for netgroup " + netgroup +
+ ": " + e.getMessage());
+ }
}
if (users != null && users.length != 0) {
return Arrays.asList(users);
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java Fri Oct 19 18:49:38 2012
@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Ch
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len > 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc >>>= 8);
- int c2 = b[off++] ^ (localCrc >>>= 8);
- int c3 = b[off++] ^ (localCrc >>>= 8);
- localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
- ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) & 0xff;
+ final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+ final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+ final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] & 0xff;
+ final int c5 = b[off+5] & 0xff;
+ final int c6 = b[off+6] & 0xff;
+ final int c7 = b[off+7] & 0xff;
- localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
- ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
- while(len > 0) {
- localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
- len--;
+
+ /* loop unroll - duff's device style */
+ switch(len) {
+ case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ default:
+ /* nothing */
}
// Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Ch
@Override
final public void update(int b) {
- crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+ crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
}
/*
* CRC-32 lookup tables generated by the polynomial 0xEDB88320.
* See also TestPureJavaCrc32.Table.
*/
- private static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256;
+ private static final int T8_2_start = 2*256;
+ private static final int T8_3_start = 3*256;
+ private static final int T8_4_start = 4*256;
+ private static final int T8_5_start = 5*256;
+ private static final int T8_6_start = 6*256;
+ private static final int T8_7_start = 7*256;
+
+ private static final int[] T = new int[] {
+ /* T8_0 */
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
@@ -154,9 +180,8 @@ public class PureJavaCrc32 implements Ch
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
- 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
- };
- private static final int[] T8_1 = new int[] {
+ 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+ /* T8_1 */
0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3,
0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7,
0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB,
@@ -220,9 +245,8 @@ public class PureJavaCrc32 implements Ch
0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E,
0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A,
0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876,
- 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
- };
- private static final int[] T8_2 = new int[] {
+ 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+ /* T8_2 */
0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59,
0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685,
0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1,
@@ -286,9 +310,8 @@ public class PureJavaCrc32 implements Ch
0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89,
0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555,
0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31,
- 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
- };
- private static final int[] T8_3 = new int[] {
+ 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+ /* T8_3 */
0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE,
0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9,
0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701,
@@ -352,9 +375,8 @@ public class PureJavaCrc32 implements Ch
0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949,
0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E,
0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6,
- 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
- };
- private static final int[] T8_4 = new int[] {
+ 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+ /* T8_4 */
0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0,
0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10,
0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111,
@@ -418,9 +440,8 @@ public class PureJavaCrc32 implements Ch
0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D,
0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D,
0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C,
- 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
- };
- private static final int[] T8_5 = new int[] {
+ 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+ /* T8_5 */
0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE,
0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8,
0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3,
@@ -484,9 +505,8 @@ public class PureJavaCrc32 implements Ch
0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7,
0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1,
0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA,
- 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
- };
- private static final int[] T8_6 = new int[] {
+ 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+ /* T8_6 */
0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D,
0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E,
0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA,
@@ -550,9 +570,8 @@ public class PureJavaCrc32 implements Ch
0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044,
0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457,
0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23,
- 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
- };
- private static final int[] T8_7 = new int[] {
+ 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+ /* T8_7 */
0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3,
0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919,
0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56,
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java Fri Oct 19 18:49:38 2012
@@ -56,22 +56,38 @@ public class PureJavaCrc32C implements C
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len > 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc >>>= 8);
- int c2 = b[off++] ^ (localCrc >>>= 8);
- int c3 = b[off++] ^ (localCrc >>>= 8);
- localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
- ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) & 0xff;
+ final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+ final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+ final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] & 0xff;
+ final int c5 = b[off+5] & 0xff;
+ final int c6 = b[off+6] & 0xff;
+ final int c7 = b[off+7] & 0xff;
- localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
- ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
- while(len > 0) {
- localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
- len--;
+
+ /* loop unroll - duff's device style */
+ switch(len) {
+ case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+ default:
+ /* nothing */
}
// Publish crc out to object
@@ -80,14 +96,24 @@ public class PureJavaCrc32C implements C
@Override
final public void update(int b) {
- crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+ crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
}
// CRC polynomial tables generated by:
// java -cp build/test/classes/:build/classes/ \
// org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78
- static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256;
+ private static final int T8_2_start = 2*256;
+ private static final int T8_3_start = 3*256;
+ private static final int T8_4_start = 4*256;
+ private static final int T8_5_start = 5*256;
+ private static final int T8_6_start = 6*256;
+ private static final int T8_7_start = 7*256;
+
+ private static final int[] T = new int[] {
+ /* T8_0 */
0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
@@ -151,9 +177,8 @@ public class PureJavaCrc32C implements C
0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
- 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
- };
- static final int[] T8_1 = new int[] {
+ 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351,
+ /* T8_1 */
0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899,
0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945,
0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21,
@@ -217,9 +242,8 @@ public class PureJavaCrc32C implements C
0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7,
0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B,
0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F,
- 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483
- };
- static final int[] T8_2 = new int[] {
+ 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483,
+ /* T8_2 */
0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073,
0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469,
0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6,
@@ -283,9 +307,8 @@ public class PureJavaCrc32C implements C
0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177,
0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D,
0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2,
- 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8
- };
- static final int[] T8_3 = new int[] {
+ 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8,
+ /* T8_3 */
0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939,
0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA,
0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF,
@@ -349,9 +372,8 @@ public class PureJavaCrc32C implements C
0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57,
0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4,
0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1,
- 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842
- };
- static final int[] T8_4 = new int[] {
+ 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842,
+ /* T8_4 */
0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4,
0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44,
0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65,
@@ -415,9 +437,8 @@ public class PureJavaCrc32C implements C
0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82,
0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532,
0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013,
- 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3
- };
- static final int[] T8_5 = new int[] {
+ 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3,
+ /* T8_5 */
0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA,
0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD,
0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5,
@@ -481,9 +502,8 @@ public class PureJavaCrc32C implements C
0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154,
0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623,
0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B,
- 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C
- };
- static final int[] T8_6 = new int[] {
+ 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C,
+ /* T8_6 */
0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558,
0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089,
0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B,
@@ -547,9 +567,8 @@ public class PureJavaCrc32C implements C
0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD,
0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C,
0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E,
- 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F
- };
- static final int[] T8_7 = new int[] {
+ 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F,
+ /* T8_7 */
0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769,
0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504,
0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3,
@@ -613,6 +632,6 @@ public class PureJavaCrc32C implements C
0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612,
0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F,
0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8,
- 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
+ 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
};
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Fri Oct 19 18:49:38 2012
@@ -608,7 +608,8 @@ public class StringUtils {
" build = " + VersionInfo.getUrl() + " -r "
+ VersionInfo.getRevision()
+ "; compiled by '" + VersionInfo.getUser()
- + "' on " + VersionInfo.getDate()}
+ + "' on " + VersionInfo.getDate(),
+ " java = " + System.getProperty("java.version") }
)
);
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Fri Oct 19 18:49:38 2012
@@ -730,6 +730,55 @@ public class TestCodec {
}
}
+ @Test
+ public void testGzipLongOverflow() throws IOException {
+ LOG.info("testGzipLongOverflow");
+
+ // Don't use native libs for this test.
+ Configuration conf = new Configuration();
+ conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
+ assertFalse("ZlibFactory is using native libs against request",
+ ZlibFactory.isNativeZlibLoaded(conf));
+
+ // Ensure that the CodecPool has a BuiltInZlibInflater in it.
+ Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
+ assertNotNull("zlibDecompressor is null!", zlibDecompressor);
+ assertTrue("ZlibFactory returned unexpected inflator",
+ zlibDecompressor instanceof BuiltInZlibInflater);
+ CodecPool.returnDecompressor(zlibDecompressor);
+
+ // Now create a GZip text file.
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+ new GZIPOutputStream(new FileOutputStream(f.toString()))));
+
+ final int NBUF = 1024 * 4 + 1;
+ final char[] buf = new char[1024 * 1024];
+ for (int i = 0; i < buf.length; i++) buf[i] = '\0';
+ for (int i = 0; i < NBUF; i++) {
+ bw.write(buf);
+ }
+ bw.close();
+
+ // Now read it back, using the CodecPool to establish the
+ // decompressor to use.
+ CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
+ CompressionCodec codec = ccf.getCodec(f);
+ Decompressor decompressor = CodecPool.getDecompressor(codec);
+ FileSystem fs = FileSystem.getLocal(conf);
+ InputStream is = fs.open(f);
+ is = codec.createInputStream(is, decompressor);
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ for (int j = 0; j < NBUF; j++) {
+ int n = br.read(buf);
+ assertEquals("got wrong read length!", n, buf.length);
+ for (int i = 0; i < buf.length; i++)
+ assertEquals("got wrong byte!", buf[i], '\0');
+ }
+ br.close();
+ }
+
public void testGzipCodecWrite(boolean useNative) throws IOException {
// Create a gzipped file using a compressor from the CodecPool,
// and try to read it back via the regular GZIPInputStream.
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java Fri Oct 19 18:49:38 2012
@@ -141,7 +141,7 @@ public class TestVLong extends TestCase
int shift = rng.nextInt(Long.SIZE) + 1;
long mask = (1L << shift) - 1;
long a = ((long) rng.nextInt()) << 32;
- long b = ((long) rng.nextInt()) & 0xffffffff;
+ long b = ((long) rng.nextInt()) & 0xffffffffL;
data[i] = (a + b) & mask;
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java Fri Oct 19 18:49:38 2012
@@ -78,5 +78,29 @@ public class TestJMXJsonServlet extends
"/jmx?get=java.lang:type=Memory::"));
LOG.info("/jmx RESULT: "+result);
assertReFind("\"ERROR\"", result);
+
+ // test to get JSONP result
+ result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory&callback=mycallback1"));
+ LOG.info("/jmx?qry=java.lang:type=Memory&callback=mycallback RESULT: "+result);
+ assertReFind("^mycallback1\\(\\{", result);
+ assertReFind("\\}\\);$", result);
+
+ // negative test to get an attribute of a mbean as JSONP
+ result = readOutput(new URL(baseUrl,
+ "/jmx?get=java.lang:type=Memory::&callback=mycallback2"));
+ LOG.info("/jmx RESULT: "+result);
+ assertReFind("^mycallback2\\(\\{", result);
+ assertReFind("\"ERROR\"", result);
+ assertReFind("\\}\\);$", result);
+
+ // test to get an attribute of a mbean as JSONP
+ result = readOutput(new URL(baseUrl,
+ "/jmx?get=java.lang:type=Memory::HeapMemoryUsage&callback=mycallback3"));
+ LOG.info("/jmx RESULT: "+result);
+ assertReFind("^mycallback3\\(\\{", result);
+ assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+ assertReFind("\"committed\"\\s*:", result);
+ assertReFind("\\}\\);$", result);
+
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java Fri Oct 19 18:49:38 2012
@@ -18,9 +18,7 @@
package org.apache.hadoop.metrics2.util;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
@@ -28,7 +26,6 @@ import java.util.Collections;
import java.util.Map;
import java.util.Random;
-import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
@@ -54,18 +51,22 @@ public class TestSampleQuantiles {
// Counts start off zero
assertEquals(estimator.getCount(), 0);
assertEquals(estimator.getSampleCount(), 0);
- try {
- estimator.snapshot();
- fail("Expected IOException from empty window");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("No samples", e);
- }
+
+ // Snapshot should be null if there are no entries.
+ assertNull(estimator.snapshot());
// Count increment correctly by 1
estimator.insert(1337);
assertEquals(estimator.getCount(), 1);
estimator.snapshot();
assertEquals(estimator.getSampleCount(), 1);
+
+ assertEquals(
+ "50.00 %ile +/- 5.00%: 1337\n" +
+ "75.00 %ile +/- 2.50%: 1337\n" +
+ "90.00 %ile +/- 1.00%: 1337\n" +
+ "95.00 %ile +/- 0.50%: 1337\n" +
+ "99.00 %ile +/- 0.10%: 1337", estimator.toString());
}
/**
@@ -80,12 +81,7 @@ public class TestSampleQuantiles {
estimator.clear();
assertEquals(estimator.getCount(), 0);
assertEquals(estimator.getSampleCount(), 0);
- try {
- estimator.snapshot();
- fail("Expected IOException for an empty window.");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("No samples", e);
- }
+ assertNull(estimator.snapshot());
}
/**
Modified: hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java?rev=1400219&r1=1400218&r2=1400219&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java Fri Oct 19 18:49:38 2012
@@ -142,16 +142,14 @@ public class TestPureJavaCrc32 {
for (int j = 0; j < tables.length; j++) {
final int[] t = tables[j];
final StringBuilder b = new StringBuilder();
- b.append(String.format(" static final int[] " + nameformat
- + " = new int[] {", j));
+ b.append(String.format(" /* "+ nameformat +" */", j));
for (int i = 0; i < t.length;) {
b.append("\n ");
for(int k = 0; k < 4; k++) {
b.append(String.format("0x%08X, ", t[i++]));
}
}
- b.setCharAt(b.length() - 2, '\n');
- s[j] = b.toString() + " };\n";
+ s[j] = b.toString();
}
return s;
}
@@ -159,10 +157,23 @@ public class TestPureJavaCrc32 {
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
- for(String s : toStrings(String.format("T%d_",
- Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
+
+ final String tableFormat = String.format("T%d_",
+ Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
+ final String startFormat = " private static final int "+tableFormat+"_start = %d*256;";
+
+ for (int j = 0; j < tables.length; j++) {
+ b.append(String.format(startFormat, j, j));
+ b.append("\n");
+ }
+
+ b.append(" private static final int[] T = new int[] {");
+ for(String s : toStrings(tableFormat)) {
+ b.append("\n");
b.append(s);
}
+ b.setCharAt(b.length() - 2, '\n');
+ b.append(" };\n");
return b.toString();
}