You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2014/07/23 03:47:33 UTC
svn commit: r1612742 - in /hadoop/common/branches/MR-2841: ./
hadoop-project/src/site/
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/
hadoop-tools/hadoop-a...
Author: todd
Date: Wed Jul 23 01:47:28 2014
New Revision: 1612742
URL: http://svn.apache.org/r1612742
Log:
Merge trunk into branch
Modified:
hadoop/common/branches/MR-2841/ (props changed)
hadoop/common/branches/MR-2841/hadoop-project/src/site/site.xml
hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
Propchange: hadoop/common/branches/MR-2841/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1610815-1612740
Modified: hadoop/common/branches/MR-2841/hadoop-project/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-project/src/site/site.xml?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/MR-2841/hadoop-project/src/site/site.xml Wed Jul 23 01:47:28 2014
@@ -89,6 +89,7 @@
<item name="HDFS NFS Gateway" href="hadoop-project-dist/hadoop-hdfs/HdfsNfsGateway.html"/>
<item name="HDFS Rolling Upgrade" href="hadoop-project-dist/hadoop-hdfs/HdfsRollingUpgrade.html"/>
<item name="Extended Attributes" href="hadoop-project-dist/hadoop-hdfs/ExtendedAttributes.html"/>
+ <item name="HDFS Support for Multihoming" href="hadoop-project-dist/hadoop-hdfs/HdfsMultihoming.html"/>
</menu>
<menu name="MapReduce" inherit="top">
Modified: hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java (original)
+++ hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java Wed Jul 23 01:47:28 2014
@@ -373,6 +373,8 @@ public class NativeAzureFileSystem exten
private Path workingDir;
private long blockSize = MAX_AZURE_BLOCK_SIZE;
private AzureFileSystemInstrumentation instrumentation;
+ private String metricsSourceName;
+ private boolean isClosed = false;
private static boolean suppressRetryPolicy = false;
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
@@ -482,11 +484,10 @@ public class NativeAzureFileSystem exten
// Make sure the metrics system is available before interacting with Azure
AzureFileSystemMetricsSystem.fileSystemStarted();
- String sourceName = newMetricsSourceName(),
- sourceDesc = "Azure Storage Volume File System metrics";
- instrumentation = DefaultMetricsSystem.instance().register(sourceName,
- sourceDesc, new AzureFileSystemInstrumentation(conf));
- AzureFileSystemMetricsSystem.registerSource(sourceName, sourceDesc,
+ metricsSourceName = newMetricsSourceName();
+ String sourceDesc = "Azure Storage Volume File System metrics";
+ instrumentation = new AzureFileSystemInstrumentation(conf);
+ AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
instrumentation);
store.initialize(uri, conf, instrumentation);
@@ -502,7 +503,6 @@ public class NativeAzureFileSystem exten
LOG.debug(" blockSize = "
+ conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE));
}
-
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
@@ -1337,7 +1337,11 @@ public class NativeAzureFileSystem exten
}
@Override
- public void close() throws IOException {
+ public synchronized void close() throws IOException {
+ if (isClosed) {
+ return;
+ }
+
// Call the base close() to close any resources there.
super.close();
// Close the store
@@ -1349,12 +1353,14 @@ public class NativeAzureFileSystem exten
long startTime = System.currentTimeMillis();
+ AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
AzureFileSystemMetricsSystem.fileSystemClosed();
if (LOG.isDebugEnabled()) {
LOG.debug("Submitting metrics when file system closed took "
+ (System.currentTimeMillis() - startTime) + " ms.");
}
+ isClosed = true;
}
/**
@@ -1498,6 +1504,13 @@ public class NativeAzureFileSystem exten
handleFilesWithDanglingTempData(root, new DanglingFileDeleter());
}
+ @Override
+ protected void finalize() throws Throwable {
+ LOG.debug("finalize() called.");
+ close();
+ super.finalize();
+ }
+
/**
* Encode the key with a random prefix for load balancing in Azure storage.
* Upload data to a random temporary file then do storage side renaming to
Modified: hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java (original)
+++ hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java Wed Jul 23 01:47:28 2014
@@ -44,21 +44,26 @@ public final class AzureFileSystemMetric
}
public static synchronized void fileSystemClosed() {
- if (instance != null) {
- instance.publishMetricsNow();
- }
if (numFileSystems == 1) {
+ instance.publishMetricsNow();
instance.stop();
instance.shutdown();
instance = null;
}
numFileSystems--;
}
-
+
public static void registerSource(String name, String desc,
MetricsSource source) {
- // Register the source with the name appended with -WasbSystem
- // so that the name is globally unique.
- instance.register(name + "-WasbSystem", desc, source);
+ //caller has to use unique name to register source
+ instance.register(name, desc, source);
+ }
+
+ public static synchronized void unregisterSource(String name) {
+ if (instance != null) {
+ //publish metrics before unregister a metrics source
+ instance.publishMetricsNow();
+ instance.unregisterSource(name);
+ }
}
}
Modified: hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java (original)
+++ hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java Wed Jul 23 01:47:28 2014
@@ -324,9 +324,7 @@ public final class AzureBlobStorageTestA
String sourceName = NativeAzureFileSystem.newMetricsSourceName();
String sourceDesc = "Azure Storage Volume File System metrics";
- AzureFileSystemInstrumentation instrumentation =
- DefaultMetricsSystem.instance().register(sourceName,
- sourceDesc, new AzureFileSystemInstrumentation(conf));
+ AzureFileSystemInstrumentation instrumentation = new AzureFileSystemInstrumentation(conf);
AzureFileSystemMetricsSystem.registerSource(
sourceName, sourceDesc, instrumentation);
Modified: hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java (original)
+++ hadoop/common/branches/MR-2841/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java Wed Jul 23 01:47:28 2014
@@ -516,6 +516,13 @@ public abstract class NativeAzureFileSys
assertNotNull(status);
}
+ @Test
+ public void testCloseFileSystemTwice() throws Exception {
+ //make sure close() can be called multiple times without doing any harm
+ fs.close();
+ fs.close();
+ }
+
private boolean testModifiedTime(Path testPath, long time) throws Exception {
FileStatus fileStatus = fs.getFileStatus(testPath);
final long errorMargin = modifiedTimeErrorMargin;
Modified: hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java (original)
+++ hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java Wed Jul 23 01:47:28 2014
@@ -162,6 +162,7 @@ public enum DistCpOptionSwitch {
BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
new Option("bandwidth", true, "Specify bandwidth per map in MB"));
+ static final String PRESERVE_STATUS_DEFAULT = "-prbugpc";
private final String confLabel;
private final Option option;
Modified: hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1612742&r1=1612741&r2=1612742&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java (original)
+++ hadoop/common/branches/MR-2841/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java Wed Jul 23 01:47:28 2014
@@ -50,7 +50,7 @@ public class OptionsParser {
protected String[] flatten(Options options, String[] arguments, boolean stopAtNonOption) {
for (int index = 0; index < arguments.length; index++) {
if (arguments[index].equals("-" + DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) {
- arguments[index] = "-prbugpc";
+ arguments[index] = DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT;
}
}
return super.flatten(options, arguments, stopAtNonOption);