You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2014/12/17 23:59:57 UTC
[11/24] hadoop git commit: HADOOP-10840. Fix OutOfMemoryError caused
by metrics system in Azure File System. Contributed by Shanyu Zhao.
HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File System. Contributed by Shanyu Zhao.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1611247 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 0a02b5a19bcca8d0c49d2adfc3909aebdf9d606e)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02a3bf8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02a3bf8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02a3bf8b
Branch: refs/heads/branch-2
Commit: 02a3bf8bda5ac8c49c45a519a35b513ce072d584
Parents: b928b8c
Author: Chris Nauroth <cn...@apache.org>
Authored: Thu Jul 17 05:04:53 2014 +0000
Committer: cnauroth <cn...@apache.org>
Committed: Wed Dec 17 14:57:12 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/fs/azure/NativeAzureFileSystem.java | 27 +++++++++++++++-----
.../metrics/AzureFileSystemMetricsSystem.java | 19 +++++++++-----
.../fs/azure/AzureBlobStorageTestAccount.java | 4 +--
.../fs/azure/NativeAzureFileSystemBaseTest.java | 7 +++++
5 files changed, 43 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02a3bf8b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d1e84cb..a40a143 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -68,6 +68,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11416. Move ChunkedArrayList into hadoop-common (cmccabe)
+ HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File
+ System. (Shanyu Zhao via cnauroth)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02a3bf8b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 2b69573..9b9fd8b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -373,6 +373,8 @@ public class NativeAzureFileSystem extends FileSystem {
private Path workingDir;
private long blockSize = MAX_AZURE_BLOCK_SIZE;
private AzureFileSystemInstrumentation instrumentation;
+ private String metricsSourceName;
+ private boolean isClosed = false;
private static boolean suppressRetryPolicy = false;
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
@@ -482,11 +484,10 @@ public class NativeAzureFileSystem extends FileSystem {
// Make sure the metrics system is available before interacting with Azure
AzureFileSystemMetricsSystem.fileSystemStarted();
- String sourceName = newMetricsSourceName(),
- sourceDesc = "Azure Storage Volume File System metrics";
- instrumentation = DefaultMetricsSystem.instance().register(sourceName,
- sourceDesc, new AzureFileSystemInstrumentation(conf));
- AzureFileSystemMetricsSystem.registerSource(sourceName, sourceDesc,
+ metricsSourceName = newMetricsSourceName();
+ String sourceDesc = "Azure Storage Volume File System metrics";
+ instrumentation = new AzureFileSystemInstrumentation(conf);
+ AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
instrumentation);
store.initialize(uri, conf, instrumentation);
@@ -502,7 +503,6 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug(" blockSize = "
+ conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE));
}
-
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
@@ -1337,7 +1337,11 @@ public class NativeAzureFileSystem extends FileSystem {
}
@Override
- public void close() throws IOException {
+ public synchronized void close() throws IOException {
+ if (isClosed) {
+ return;
+ }
+
// Call the base close() to close any resources there.
super.close();
// Close the store
@@ -1349,12 +1353,14 @@ public class NativeAzureFileSystem extends FileSystem {
long startTime = System.currentTimeMillis();
+ AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
AzureFileSystemMetricsSystem.fileSystemClosed();
if (LOG.isDebugEnabled()) {
LOG.debug("Submitting metrics when file system closed took "
+ (System.currentTimeMillis() - startTime) + " ms.");
}
+ isClosed = true;
}
/**
@@ -1498,6 +1504,13 @@ public class NativeAzureFileSystem extends FileSystem {
handleFilesWithDanglingTempData(root, new DanglingFileDeleter());
}
+ @Override
+ protected void finalize() throws Throwable {
+ LOG.debug("finalize() called.");
+ close();
+ super.finalize();
+ }
+
/**
* Encode the key with a random prefix for load balancing in Azure storage.
* Upload data to a random temporary file then do storage side renaming to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02a3bf8b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
index a5f29c1..322795a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
@@ -44,21 +44,26 @@ public final class AzureFileSystemMetricsSystem {
}
public static synchronized void fileSystemClosed() {
- if (instance != null) {
- instance.publishMetricsNow();
- }
if (numFileSystems == 1) {
+ instance.publishMetricsNow();
instance.stop();
instance.shutdown();
instance = null;
}
numFileSystems--;
}
-
+
public static void registerSource(String name, String desc,
MetricsSource source) {
- // Register the source with the name appended with -WasbSystem
- // so that the name is globally unique.
- instance.register(name + "-WasbSystem", desc, source);
+ //caller has to use unique name to register source
+ instance.register(name, desc, source);
+ }
+
+ public static synchronized void unregisterSource(String name) {
+ if (instance != null) {
+ //publish metrics before unregister a metrics source
+ instance.publishMetricsNow();
+ instance.unregisterSource(name);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02a3bf8b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 02738e7..80e8e43 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -324,9 +324,7 @@ public final class AzureBlobStorageTestAccount {
String sourceName = NativeAzureFileSystem.newMetricsSourceName();
String sourceDesc = "Azure Storage Volume File System metrics";
- AzureFileSystemInstrumentation instrumentation =
- DefaultMetricsSystem.instance().register(sourceName,
- sourceDesc, new AzureFileSystemInstrumentation(conf));
+ AzureFileSystemInstrumentation instrumentation = new AzureFileSystemInstrumentation(conf);
AzureFileSystemMetricsSystem.registerSource(
sourceName, sourceDesc, instrumentation);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02a3bf8b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
index bc7e344..e731b21 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
@@ -516,6 +516,13 @@ public abstract class NativeAzureFileSystemBaseTest {
assertNotNull(status);
}
+ @Test
+ public void testCloseFileSystemTwice() throws Exception {
+ //make sure close() can be called multiple times without doing any harm
+ fs.close();
+ fs.close();
+ }
+
private boolean testModifiedTime(Path testPath, long time) throws Exception {
FileStatus fileStatus = fs.getFileStatus(testPath);
final long errorMargin = modifiedTimeErrorMargin;