You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/11/04 13:08:55 UTC
[6/9] git commit: HADOOP-11248. Add hadoop configuration to disable
Azure Filesystem metrics collection. Contributed by Shanyu Zhao.
HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. Contributed by Shanyu Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/734eeb4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/734eeb4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/734eeb4f
Branch: refs/heads/HDFS-EC
Commit: 734eeb4f357ad3210355a0d3fdbc80706a770d61
Parents: ec6cbec
Author: cnauroth <cn...@apache.org>
Authored: Mon Nov 3 14:29:18 2014 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Mon Nov 3 14:29:18 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/fs/azure/NativeAzureFileSystem.java | 20 +++++++++++++-------
.../TestNativeAzureFileSystemMetricsSystem.java | 11 +++++++++++
3 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 93e58f6..eb91dcb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -339,6 +339,9 @@ Trunk (Unreleased)
HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g.,
sbin) (aw)
+ HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics
+ collection. (Shanyu Zhao via cnauroth)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 076c48a..ad2e2e6 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -641,6 +641,8 @@ public class NativeAzureFileSystem extends FileSystem {
static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME =
"fs.azure.output.stream.buffer.size";
+ public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics";
+
private class NativeAzureFsInputStream extends FSInputStream {
private InputStream in;
private final String key;
@@ -1035,13 +1037,15 @@ public class NativeAzureFileSystem extends FileSystem {
store = createDefaultStore(conf);
}
- // Make sure the metrics system is available before interacting with Azure
- AzureFileSystemMetricsSystem.fileSystemStarted();
- metricsSourceName = newMetricsSourceName();
- String sourceDesc = "Azure Storage Volume File System metrics";
instrumentation = new AzureFileSystemInstrumentation(conf);
- AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
+ if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
+ // Make sure the metrics system is available before interacting with Azure
+ AzureFileSystemMetricsSystem.fileSystemStarted();
+ metricsSourceName = newMetricsSourceName();
+ String sourceDesc = "Azure Storage Volume File System metrics";
+ AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
instrumentation);
+ }
store.initialize(uri, conf, instrumentation);
setConf(conf);
@@ -2207,8 +2211,10 @@ public class NativeAzureFileSystem extends FileSystem {
long startTime = System.currentTimeMillis();
- AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
- AzureFileSystemMetricsSystem.fileSystemClosed();
+ if(!getConf().getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
+ AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
+ AzureFileSystemMetricsSystem.fileSystemClosed();
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("Submitting metrics when file system closed took "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
index f44613d..7820b7e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
@@ -74,4 +74,15 @@ public class TestNativeAzureFileSystemMetricsSystem {
assertTrue(name2.startsWith("AzureFileSystemMetrics"));
assertTrue(!name1.equals(name2));
}
+
+ @Test
+ public void testSkipMetricsCollection() throws Exception {
+ AzureBlobStorageTestAccount a;
+ a = AzureBlobStorageTestAccount.createMock();
+ a.getFileSystem().getConf().setBoolean(
+ NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, true);
+ a.getFileSystem().create(new Path("/foo")).close();
+ a.closeFileSystem(); // Causes the file system to close, which publishes metrics
+ assertEquals(0, getFilesCreated(a));
+ }
}