You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tm...@apache.org on 2018/08/11 05:37:50 UTC
[46/50] [abbrv] hadoop git commit: HADOOP-15407. HADOOP-15540.
Support Windows Azure Storage - Blob file system "ABFS" in Hadoop: Core
Commit.
HADOOP-15407. HADOOP-15540. Support Windows Azure Storage - Blob file system "ABFS" in Hadoop: Core Commit.
Contributed by Shane Mainali, Thomas Marquardt, Zichen Sun, Georgi Chalakov, Esfandiar Manii, Amit Singh, Dana Kaban, Da Zhou, Junhua Gu, Saher Ahwal, Saurabh Pant, James Baker, Shaoyu Zhang, Lawrence Chen, Kevin Chen and Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdf5f4c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdf5f4c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdf5f4c3
Branch: refs/heads/HADOOP-15407
Commit: fdf5f4c3243e8c8d76f443cb15aeb19a4edb4bb6
Parents: a2a8c48
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jun 15 18:14:13 2018 +0100
Committer: Thomas Marquardt <tm...@microsoft.com>
Committed: Sat Aug 11 03:42:27 2018 +0000
----------------------------------------------------------------------
.gitignore | 1 +
.../src/main/resources/core-default.xml | 10 +
.../conf/TestCommonConfigurationFields.java | 3 +
hadoop-project/pom.xml | 11 +
hadoop-tools/hadoop-azure/pom.xml | 61 +-
.../src/config/checkstyle-suppressions.xml | 47 ++
.../org/apache/hadoop/fs/azurebfs/Abfs.java | 48 ++
.../org/apache/hadoop/fs/azurebfs/Abfss.java | 48 ++
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 612 ++++++++++++++++
.../fs/azurebfs/SecureAzureBlobFileSystem.java | 41 ++
.../azurebfs/constants/AbfsHttpConstants.java | 76 ++
.../azurebfs/constants/ConfigurationKeys.java | 57 ++
.../constants/FileSystemConfigurations.java | 59 ++
.../constants/FileSystemUriSchemes.java | 42 ++
.../constants/HttpHeaderConfigurations.java | 57 ++
.../fs/azurebfs/constants/HttpQueryParams.java | 40 ++
.../fs/azurebfs/constants/package-info.java | 22 +
.../ConfigurationValidationAnnotations.java | 104 +++
.../contracts/annotations/package-info.java | 22 +
.../diagnostics/ConfigurationValidator.java | 37 +
.../contracts/diagnostics/package-info.java | 22 +
.../exceptions/AbfsRestOperationException.java | 84 +++
.../AzureBlobFileSystemException.java | 56 ++
.../ConfigurationPropertyNotFoundException.java | 32 +
.../FileSystemOperationUnhandledException.java | 33 +
.../InvalidAbfsRestOperationException.java | 40 ++
.../InvalidConfigurationValueException.java | 37 +
.../InvalidFileSystemPropertyException.java | 33 +
.../InvalidUriAuthorityException.java | 33 +
.../exceptions/InvalidUriException.java | 33 +
.../exceptions/ServiceResolutionException.java | 36 +
.../contracts/exceptions/TimeoutException.java | 33 +
.../contracts/exceptions/package-info.java | 22 +
.../fs/azurebfs/contracts/package-info.java | 22 +
.../services/AbfsHttpClientFactory.java | 39 ++
.../contracts/services/AbfsHttpService.java | 162 +++++
.../contracts/services/AbfsServiceProvider.java | 40 ++
.../services/AzureServiceErrorCode.java | 112 +++
.../services/ConfigurationService.java | 143 ++++
.../contracts/services/InjectableService.java | 30 +
.../services/ListResultEntrySchema.java | 160 +++++
.../contracts/services/ListResultSchema.java | 58 ++
.../contracts/services/ReadBufferStatus.java | 29 +
.../contracts/services/TracingService.java | 66 ++
.../contracts/services/package-info.java | 22 +
...Base64StringConfigurationBasicValidator.java | 50 ++
.../BooleanConfigurationBasicValidator.java | 50 ++
.../ConfigurationBasicValidator.java | 67 ++
.../IntegerConfigurationBasicValidator.java | 68 ++
.../LongConfigurationBasicValidator.java | 65 ++
.../StringConfigurationBasicValidator.java | 45 ++
.../fs/azurebfs/diagnostics/package-info.java | 22 +
.../org/apache/hadoop/fs/azurebfs/package.html | 31 +
.../hadoop/fs/azurebfs/services/AbfsClient.java | 402 +++++++++++
.../services/AbfsHttpClientFactoryImpl.java | 116 ++++
.../fs/azurebfs/services/AbfsHttpHeader.java | 40 ++
.../fs/azurebfs/services/AbfsHttpOperation.java | 430 ++++++++++++
.../azurebfs/services/AbfsHttpServiceImpl.java | 693 +++++++++++++++++++
.../fs/azurebfs/services/AbfsInputStream.java | 382 ++++++++++
.../fs/azurebfs/services/AbfsOutputStream.java | 335 +++++++++
.../fs/azurebfs/services/AbfsRestOperation.java | 178 +++++
.../services/AbfsServiceInjectorImpl.java | 81 +++
.../services/AbfsServiceProviderImpl.java | 96 +++
.../azurebfs/services/AbfsUriQueryBuilder.java | 58 ++
.../services/ConfigurationServiceImpl.java | 317 +++++++++
.../services/ExponentialRetryPolicy.java | 141 ++++
.../azurebfs/services/LoggerSpanReceiver.java | 74 ++
.../hadoop/fs/azurebfs/services/ReadBuffer.java | 139 ++++
.../fs/azurebfs/services/ReadBufferManager.java | 391 +++++++++++
.../fs/azurebfs/services/ReadBufferWorker.java | 72 ++
.../azurebfs/services/SharedKeyCredentials.java | 507 ++++++++++++++
.../azurebfs/services/TracingServiceImpl.java | 134 ++++
.../fs/azurebfs/services/package-info.java | 22 +
.../hadoop/fs/azurebfs/utils/UriUtils.java | 77 +++
.../hadoop/fs/azurebfs/utils/package-info.java | 22 +
.../fs/azurebfs/DependencyInjectedTest.java | 225 ++++++
.../ITestAzureBlobFileSystemAppend.java | 78 +++
.../ITestAzureBlobFileSystemBackCompat.java | 83 +++
.../azurebfs/ITestAzureBlobFileSystemCopy.java | 100 +++
.../ITestAzureBlobFileSystemCreate.java | 110 +++
.../ITestAzureBlobFileSystemDelete.java | 119 ++++
.../azurebfs/ITestAzureBlobFileSystemE2E.java | 150 ++++
.../ITestAzureBlobFileSystemE2EScale.java | 133 ++++
.../ITestAzureBlobFileSystemFileStatus.java | 64 ++
.../azurebfs/ITestAzureBlobFileSystemFlush.java | 204 ++++++
.../ITestAzureBlobFileSystemInitAndCreate.java | 50 ++
.../ITestAzureBlobFileSystemListStatus.java | 132 ++++
.../azurebfs/ITestAzureBlobFileSystemMkDir.java | 88 +++
.../azurebfs/ITestAzureBlobFileSystemOpen.java | 41 ++
.../ITestAzureBlobFileSystemRandomRead.java | 582 ++++++++++++++++
.../ITestAzureBlobFileSystemRename.java | 152 ++++
.../azurebfs/ITestFileSystemInitialization.java | 78 +++
.../azurebfs/ITestFileSystemRegistration.java | 82 +++
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 202 ++++++
.../constants/TestConfigurationKeys.java | 37 +
.../fs/azurebfs/constants/package-info.java | 22 +
.../DependencyInjectedContractTest.java | 63 ++
.../contract/ITestAbfsFileSystemContract.java | 54 ++
.../ITestAbfsFileSystemContractAppend.java | 70 ++
.../ITestAbfsFileSystemContractConcat.java | 62 ++
.../ITestAbfsFileSystemContractCreate.java | 63 ++
.../ITestAbfsFileSystemContractDelete.java | 63 ++
.../ITestAbfsFileSystemContractDistCp.java | 44 ++
...TestAbfsFileSystemContractGetFileStatus.java | 62 ++
.../ITestAbfsFileSystemContractMkdir.java | 63 ++
.../ITestAbfsFileSystemContractOpen.java | 63 ++
.../ITestAbfsFileSystemContractRename.java | 63 ++
...TestAbfsFileSystemContractRootDirectory.java | 67 ++
...ITestAbfsFileSystemContractSecureDistCp.java | 44 ++
.../ITestAbfsFileSystemContractSeek.java | 63 ++
.../ITestAbfsFileSystemContractSetTimes.java | 62 ++
.../ITestAzureBlobFileSystemBasics.java | 91 +++
.../fs/azurebfs/contract/package-info.java | 22 +
.../TestConfigurationValidators.java | 124 ++++
.../fs/azurebfs/diagnostics/package-info.java | 22 +
.../apache/hadoop/fs/azurebfs/package-info.java | 22 +
.../services/ITestAbfsHttpServiceImpl.java | 122 ++++
.../services/ITestReadWriteAndSeek.java | 80 +++
.../services/ITestTracingServiceImpl.java | 79 +++
.../services/MockAbfsHttpClientFactoryImpl.java | 69 ++
.../services/MockAbfsServiceInjectorImpl.java | 50 ++
.../services/MockServiceProviderImpl.java | 36 +
...estConfigurationServiceFieldsValidation.java | 149 ++++
.../fs/azurebfs/services/package-info.java | 22 +
.../hadoop/fs/azurebfs/utils/TestUriUtils.java | 48 ++
.../hadoop/fs/azurebfs/utils/package-info.java | 22 +
.../hadoop-azure/src/test/resources/abfs.xml | 64 ++
.../src/test/resources/azure-bfs-test.xml | 76 ++
.../src/test/resources/log4j.properties | 2 +
129 files changed, 12606 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index d555036..ac245c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,7 @@ make-build-debug
# Filesystem contract test options and credentials
auth-keys.xml
azure-auth-keys.xml
+azure-bfs-auth-keys.xml
# External tool builders
*/.externalToolBuilders
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 29c2bc2..93b8ddd 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1616,6 +1616,16 @@
</description>
</property>
<property>
+ <name>fs.abfs.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value>
+ <description>The implementation class of the Azure Blob Filesystem</description>
+</property>
+<property>
+ <name>fs.abfss.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value>
+ <description>The implementation class of the Secure Azure Blob Filesystem</description>
+</property>
+<property>
<name>fs.azure.local.sas.key.mode</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 023c831..e10617d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -113,6 +113,9 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPrefixToSkipCompare.add("fs.wasb.impl");
xmlPrefixToSkipCompare.add("fs.wasbs.impl");
xmlPrefixToSkipCompare.add("fs.azure.");
+ xmlPrefixToSkipCompare.add("fs.abfs.impl");
+ xmlPrefixToSkipCompare.add("fs.abfss.impl");
+
// ADL properties are in a different subtree
// - org.apache.hadoop.hdfs.web.ADLConfKeys
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7603842..64aa43e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1207,6 +1207,11 @@
</dependency>
<dependency>
<groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ <version>3.1.0-incubating</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.htrace</groupId>
<artifactId>htrace-core4</artifactId>
<version>4.1.0-incubating</version>
</dependency>
@@ -1341,6 +1346,12 @@
</dependency>
<dependency>
+ <groupId>org.threadly</groupId>
+ <artifactId>threadly</artifactId>
+ <version>4.9.0</version>
+ </dependency>
+
+ <dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>2.8.3</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 52b5b72..d4046ef 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -67,6 +67,7 @@
<!-- To run with the default Sun ruleset,
comment out the configLocation line -->
<configLocation>src/config/checkstyle.xml</configLocation>
+ <suppressionsLocation>src/config/checkstyle-suppressions.xml</suppressionsLocation>
</configuration>
</plugin>
@@ -149,10 +150,22 @@
</dependency>
<dependency>
+ <groupId>org.threadly</groupId>
+ <artifactId>threadly</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<scope>compile</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>compile</scope>
+ </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -173,8 +186,38 @@
</dependency>
<dependency>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core4</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <scope>compile</scope>
+ <!-- we have a dependency on a lower version -->
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
<scope>compile</scope>
</dependency>
@@ -183,15 +226,8 @@
<artifactId>jetty-util-ajax</artifactId>
<scope>compile</scope>
</dependency>
-
-
- <!-- dependencies use for test only -->
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <scope>test</scope>
- </dependency>
+ <!-- dependencies use for test only -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -229,10 +265,6 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
</dependencies>
@@ -398,8 +430,10 @@
<exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude>
<exclude>**/ITestNativeAzureFileSystemLive.java</exclude>
<exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude>
+ <exclude>**/ITestAzureBlobFileSystemRandomRead.java</exclude>
<exclude>**/ITestWasbRemoteCallHelper.java</exclude>
<exclude>**/ITestBlockBlobInputStream.java</exclude>
+ <exclude>**/ITestWasbAbfsCompatibility.java</exclude>
</excludes>
</configuration>
</execution>
@@ -424,12 +458,15 @@
<fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
</systemPropertyVariables>
<includes>
+ <include>**/ITestWasbAbfsCompatibility.java</include>
<include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
<include>**/ITestFileSystemOperationsWithThreads.java</include>
<include>**/ITestOutOfBandAzureBlobOperationsLive.java</include>
<include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include>
<include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include>
<include>**/ITestNativeAzureFileSystemLive.java</include>
+ <include>**/ITestNativeAzureFSPageBlobLive.java</include>
+ <include>**/ITestAzureBlobFileSystemRandomRead.java</include>
<include>**/ITestWasbRemoteCallHelper.java</include>
<include>**/ITestBlockBlobInputStream.java</include>
</includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
new file mode 100644
index 0000000..0204355
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE suppressions PUBLIC
+ "-//Puppy Crawl//DTD Suppressions 1.0//EN"
+ "http://www.puppycrawl.com/dtds/suppressions_1_0.dtd">
+
+
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Checkstyle configuration that checks the sun coding conventions from:
+
+ - the Java Language Specification at
+ http://java.sun.com/docs/books/jls/second_edition/html/index.html
+
+ - the Sun Code Conventions at http://java.sun.com/docs/codeconv/
+
+ - the Javadoc guidelines at
+ http://java.sun.com/j2se/javadoc/writingdoccomments/index.html
+
+ - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html
+
+ - some best practices
+
+ Checkstyle is very configurable. Be sure to read the documentation at
+ http://checkstyle.sf.net (or in your downloaded distribution).
+
+ Most Checks are configurable, be sure to consult the documentation.
+ To completely disable a check, just comment it out or delete it from the file.
+ Finally, it is worth reading the documentation.
+-->
+
+<suppressions>
+ <suppress checks="ParameterNumber|MagicNumber"
+ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]services[\\/]AbfsHttpServiceImpl.java"/>
+</suppressions>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java
new file mode 100644
index 0000000..707e264
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+
+/**
+ * Azure Blob File System implementation of AbstractFileSystem.
+ * This impl delegates to the old FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Abfs extends DelegateToFileSystem {
+
+ Abfs(final URI theUri, final Configuration conf) throws IOException,
+ URISyntaxException {
+ super(theUri, new AzureBlobFileSystem(), conf, FileSystemUriSchemes.ABFS_SCHEME, false);
+ }
+
+ @Override
+ public int getUriDefaultPort() {
+ return -1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java
new file mode 100644
index 0000000..19c0f7a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+
+/**
+ * Azure Blob File System implementation of AbstractFileSystem.
+ * This impl delegates to the old FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Abfss extends DelegateToFileSystem {
+
+ Abfss(final URI theUri, final Configuration conf) throws IOException,
+ URISyntaxException {
+ super(theUri, new SecureAzureBlobFileSystem(), conf, FileSystemUriSchemes.ABFS_SECURE_SCHEME, false);
+ }
+
+ @Override
+ public int getUriDefaultPort() {
+ return -1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
new file mode 100644
index 0000000..707c81e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -0,0 +1,612 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.threadly.util.ExceptionUtils;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.services.AbfsServiceProviderImpl;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.contracts.services.TracingService;
+import org.apache.hadoop.fs.azurebfs.contracts.services.ConfigurationService;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsHttpService;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsServiceProvider;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.FileSystemOperationUnhandledException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.apache.htrace.core.TraceScope;
+
+/**
+ * A {@link org.apache.hadoop.fs.FileSystem} for reading and writing files stored on <a
+ * href="http://store.azure.com/">Windows Azure</a>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class AzureBlobFileSystem extends FileSystem {
+ public static final Logger LOG = LoggerFactory.getLogger(AzureBlobFileSystem.class);
+ private URI uri;
+ private Path workingDir;
+ private UserGroupInformation userGroupInformation;
+ private String user;
+ private String primaryUserGroup;
+ private AbfsServiceProvider abfsServiceProvider;
+ private TracingService tracingService;
+ private AbfsHttpService abfsHttpService;
+ private ConfigurationService configurationService;
+ private boolean isClosed;
+
+ @Override
+ public void initialize(URI uri, Configuration configuration)
+ throws IOException {
+ uri = ensureAuthority(uri, configuration);
+ super.initialize(uri, configuration);
+
+ setConf(configuration);
+
+ try {
+ this.abfsServiceProvider = AbfsServiceProviderImpl.create(configuration);
+ this.tracingService = abfsServiceProvider.get(TracingService.class);
+ this.abfsHttpService = abfsServiceProvider.get(AbfsHttpService.class);
+ this.configurationService = abfsServiceProvider.get(ConfigurationService.class);
+ } catch (AzureBlobFileSystemException exception) {
+ throw new IOException(exception);
+ }
+
+ this.LOG.debug(
+ "Initializing AzureBlobFileSystem for {}", uri);
+
+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+ this.userGroupInformation = UserGroupInformation.getCurrentUser();
+ this.user = userGroupInformation.getUserName();
+ this.primaryUserGroup = userGroupInformation.getPrimaryGroupName();
+
+ this.LOG.debug(
+ "Initializing NativeAzureFileSystem for {}", uri);
+
+ this.setWorkingDirectory(this.getHomeDirectory());
+
+ if (this.configurationService.getCreateRemoteFileSystemDuringInitialization()) {
+ this.createFileSystem();
+ }
+
+ this.mkdirs(this.workingDir);
+ }
+
+ public boolean isSecure() {
+ return false;
+ }
+
+ @Override
+ public URI getUri() {
+ return this.uri;
+ }
+
+ @Override
+ public FSDataInputStream open(final Path path, final int bufferSize) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.open path: {} bufferSize: {}", path.toString(), bufferSize);
+
+ try {
+ InputStream inputStream = abfsHttpService.openFileForRead(this, makeQualified(path), statistics);
+ return new FSDataInputStream(inputStream);
+ } catch(AzureBlobFileSystemException ex) {
+ checkException(path, ex);
+ return null;
+ }
+ }
+
+ @Override
+ public FSDataOutputStream create(final Path f, final FsPermission permission, final boolean overwrite, final int bufferSize,
+ final short replication, final long blockSize, final Progressable progress) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.create path: {} permission: {} overwrite: {} bufferSize: {}",
+ f.toString(),
+ permission,
+ overwrite,
+ blockSize);
+
+ try {
+ OutputStream outputStream = abfsHttpService.createFile(this, makeQualified(f), overwrite);
+ return new FSDataOutputStream(outputStream, statistics);
+ } catch(AzureBlobFileSystemException ex) {
+ checkException(f, ex);
+ return null;
+ }
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public FSDataOutputStream createNonRecursive(final Path f, final FsPermission permission,
+ final boolean overwrite, final int bufferSize, final short replication, final long blockSize,
+ final Progressable progress) throws IOException {
+
+ final Path parent = f.getParent();
+ final FileStatus parentFileStatus = tryGetFileStatus(parent);
+
+ if (parentFileStatus == null) {
+ throw new FileNotFoundException("Cannot create file "
+ + f.getName() + " because parent folder does not exist.");
+ }
+
+ return create(f, permission, overwrite, bufferSize, replication, blockSize, progress);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public FSDataOutputStream createNonRecursive(final Path f, final FsPermission permission,
+ final EnumSet<CreateFlag> flags, final int bufferSize, final short replication, final long blockSize,
+ final Progressable progress) throws IOException {
+
+ // Check if file should be appended or overwritten. Assume that the file
+ // is overwritten on if the CREATE and OVERWRITE create flags are set.
+ final EnumSet<CreateFlag> createflags =
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
+ final boolean overwrite = flags.containsAll(createflags);
+
+ // Delegate the create non-recursive call.
+ return this.createNonRecursive(f, permission, overwrite,
+ bufferSize, replication, blockSize, progress);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public FSDataOutputStream createNonRecursive(final Path f,
+ final boolean overwrite, final int bufferSize, final short replication, final long blockSize,
+ final Progressable progress) throws IOException {
+ return this.createNonRecursive(f, FsPermission.getFileDefault(),
+ overwrite, bufferSize, replication, blockSize, progress);
+ }
+
+ @Override
+ public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.append path: {} bufferSize: {}",
+ f.toString(),
+ bufferSize);
+
+ try {
+ OutputStream outputStream = abfsHttpService.openFileForWrite(this, makeQualified(f), false);
+ return new FSDataOutputStream(outputStream, statistics);
+ } catch(AzureBlobFileSystemException ex) {
+ checkException(f, ex);
+ return null;
+ }
+ }
+
+ public boolean rename(final Path src, final Path dst) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.rename src: {} dst: {}", src.toString(), dst.toString());
+
+ Path parentFolder = src.getParent();
+ if (parentFolder == null) {
+ return false;
+ }
+
+ final FileStatus dstFileStatus = tryGetFileStatus(dst);
+ try {
+ String sourceFileName = src.getName();
+ Path adjustedDst = dst;
+
+ if (dstFileStatus != null) {
+ if (!dstFileStatus.isDirectory()) {
+ return src.equals(dst);
+ }
+
+ adjustedDst = new Path(dst, sourceFileName);
+ }
+
+ abfsHttpService.rename(this, makeQualified(src), makeQualified(adjustedDst));
+ return true;
+ } catch(AzureBlobFileSystemException ex) {
+ checkException(
+ src,
+ ex,
+ AzureServiceErrorCode.PATH_ALREADY_EXISTS,
+ AzureServiceErrorCode.INVALID_RENAME_SOURCE_PATH,
+ AzureServiceErrorCode.SOURCE_PATH_NOT_FOUND,
+ AzureServiceErrorCode.INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE,
+ AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND);
+ return false;
+ }
+
+ }
+
+ @Override
+ public boolean delete(final Path f, final boolean recursive) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.delete path: {} recursive: {}", f.toString(), recursive);
+
+ if (f.isRoot()) {
+ if (!recursive) {
+ return false;
+ }
+
+ return deleteRoot();
+ }
+
+ try {
+ abfsHttpService.delete(this, makeQualified(f), recursive);
+ return true;
+ } catch (AzureBlobFileSystemException ex) {
+ checkException(f, ex, AzureServiceErrorCode.PATH_NOT_FOUND);
+ return false;
+ }
+
+ }
+
+ @Override
+ public FileStatus[] listStatus(final Path f) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.listStatus path: {}", f.toString());
+
+ try {
+ FileStatus[] result = abfsHttpService.listStatus(this, makeQualified(f));
+ return result;
+ } catch (AzureBlobFileSystemException ex) {
+ checkException(f, ex);
+ return null;
+ }
+ }
+
+ @Override
+ public boolean mkdirs(final Path f, final FsPermission permission) throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.mkdirs path: {} permissions: {}", f.toString(), permission);
+
+ final Path parentFolder = f.getParent();
+ if (parentFolder == null) {
+ // Cannot create root
+ return true;
+ }
+
+ try {
+ abfsHttpService.createDirectory(this, makeQualified(f));
+ return true;
+ } catch (AzureBlobFileSystemException ex) {
+ checkException(f, ex, AzureServiceErrorCode.PATH_ALREADY_EXISTS);
+ return true;
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (isClosed) {
+ return;
+ }
+
+ super.close();
+ this.LOG.debug("AzureBlobFileSystem.close");
+
+ try {
+ abfsHttpService.closeFileSystem(this);
+ } catch (AzureBlobFileSystemException ex) {
+ checkException(null, ex);
+ this.isClosed = true;
+ }
+ }
+
+ @Override
+ public FileStatus getFileStatus(final Path f) throws IOException {
+ this.LOG.debug("AzureBlobFileSystem.getFileStatus path: {}", f.toString());
+
+ try {
+ return abfsHttpService.getFileStatus(this, makeQualified(f));
+ } catch(AzureBlobFileSystemException ex) {
+ checkException(f, ex);
+ return null;
+ }
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return this.workingDir;
+ }
+
+ @Override
+ public void setWorkingDirectory(final Path newDir) {
+ if (newDir.isAbsolute()) {
+ this.workingDir = newDir;
+ } else {
+ this.workingDir = new Path(workingDir, newDir);
+ }
+ }
+
+ @Override
+ public String getScheme() {
+ return FileSystemUriSchemes.ABFS_SCHEME;
+ }
+
+ @Override
+ public Path getHomeDirectory() {
+ return makeQualified(new Path(
+ FileSystemConfigurations.USER_HOME_DIRECTORY_PREFIX + "/" + this.userGroupInformation.getShortUserName()));
+ }
+
+ /**
+ * Return an array containing hostnames, offset and size of
+ * portions of the given file. For ABFS we'll just lie and give
+ * fake hosts to make sure we get many splits in MR jobs.
+ */
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus file,
+ long start, long len) throws IOException {
+ if (file == null) {
+ return null;
+ }
+
+ if ((start < 0) || (len < 0)) {
+ throw new IllegalArgumentException("Invalid start or len parameter");
+ }
+
+ if (file.getLen() < start) {
+ return new BlockLocation[0];
+ }
+ final String blobLocationHost = this.configurationService.getAzureBlockLocationHost();
+
+ final String[] name = { blobLocationHost };
+ final String[] host = { blobLocationHost };
+ long blockSize = file.getBlockSize();
+ if (blockSize <= 0) {
+ throw new IllegalArgumentException(
+ "The block size for the given file is not a positive number: "
+ + blockSize);
+ }
+ int numberOfLocations = (int) (len / blockSize)
+ + ((len % blockSize == 0) ? 0 : 1);
+ BlockLocation[] locations = new BlockLocation[numberOfLocations];
+ for (int i = 0; i < locations.length; i++) {
+ long currentOffset = start + (i * blockSize);
+ long currentLength = Math.min(blockSize, start + len - currentOffset);
+ locations[i] = new BlockLocation(name, host, currentOffset, currentLength);
+ }
+
+ return locations;
+ }
+
+ public String getOwnerUser() {
+ return user;
+ }
+
+ public String getOwnerUserPrimaryGroup() {
+ return primaryUserGroup;
+ }
+
+ private boolean deleteRoot() throws IOException {
+ this.LOG.debug("Deleting root content");
+
+ final ExecutorService executorService = Executors.newFixedThreadPool(10);
+
+ try {
+ final FileStatus[] ls = listStatus(makeQualified(new Path(File.separator)));
+ final ArrayList<Future> deleteTasks = new ArrayList<>();
+ for (final FileStatus fs : ls) {
+ final Future deleteTask = executorService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ delete(fs.getPath(), fs.isDirectory());
+ return null;
+ }
+ });
+ deleteTasks.add(deleteTask);
+ }
+
+ for (final Future deleteTask : deleteTasks) {
+ execute("deleteRoot", new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ deleteTask.get();
+ return null;
+ }
+ });
+ }
+ }
+ finally {
+ executorService.shutdownNow();
+ }
+
+ return true;
+ }
+
+ private FileStatus tryGetFileStatus(final Path f) {
+ try {
+ return getFileStatus(f);
+ }
+ catch (IOException ex) {
+ this.LOG.debug("File not found {}", f.toString());
+ return null;
+ }
+ }
+
+ private void createFileSystem() throws IOException {
+ this.LOG.debug(
+ "AzureBlobFileSystem.createFileSystem uri: {}", uri);
+ try {
+ abfsHttpService.createFilesystem(this);
+ } catch (AzureBlobFileSystemException ex) {
+ checkException(null, ex, AzureServiceErrorCode.FILE_SYSTEM_ALREADY_EXISTS);
+ }
+
+
+ }
+
+ private URI ensureAuthority(URI uri, final Configuration conf) {
+
+ Preconditions.checkNotNull(uri, "uri");
+
+ if (uri.getAuthority() == null) {
+ final URI defaultUri = FileSystem.getDefaultUri(conf);
+
+ if (defaultUri != null && isAbfsScheme(defaultUri.getScheme())) {
+ try {
+ // Reconstruct the URI with the authority from the default URI.
+ uri = new URI(
+ uri.getScheme(),
+ defaultUri.getAuthority(),
+ uri.getPath(),
+ uri.getQuery(),
+ uri.getFragment());
+ } catch (URISyntaxException e) {
+ // This should never happen.
+ throw new IllegalArgumentException(new InvalidUriException(uri.toString()));
+ }
+ }
+ }
+
+ if (uri.getAuthority() == null) {
+ throw new IllegalArgumentException(new InvalidUriAuthorityException(uri.toString()));
+ }
+
+ return uri;
+ }
+
+ private boolean isAbfsScheme(final String scheme) {
+ if (scheme == null) {
+ return false;
+ }
+
+ if (scheme.equals(FileSystemUriSchemes.ABFS_SCHEME) || scheme.equals(FileSystemUriSchemes.ABFS_SECURE_SCHEME)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ @VisibleForTesting
+ <T> FileSystemOperation execute(
+ final String scopeDescription,
+ final Callable<T> callableFileOperation) throws IOException {
+ return execute(scopeDescription, callableFileOperation, null);
+ }
+
+ @VisibleForTesting
+ <T> FileSystemOperation execute(
+ final String scopeDescription,
+ final Callable<T> callableFileOperation,
+ T defaultResultValue) throws IOException {
+
+ final TraceScope traceScope = tracingService.traceBegin(scopeDescription);
+ try {
+ final T executionResult = callableFileOperation.call();
+ return new FileSystemOperation(executionResult, null);
+ } catch (AbfsRestOperationException abfsRestOperationException) {
+ return new FileSystemOperation(defaultResultValue, abfsRestOperationException);
+ } catch (AzureBlobFileSystemException azureBlobFileSystemException) {
+ tracingService.traceException(traceScope, azureBlobFileSystemException);
+ throw new IOException(azureBlobFileSystemException);
+ } catch (Exception exception) {
+ if (exception instanceof ExecutionException) {
+ exception = (Exception) ExceptionUtils.getRootCause(exception);
+ }
+
+ final FileSystemOperationUnhandledException fileSystemOperationUnhandledException = new FileSystemOperationUnhandledException(exception);
+ tracingService.traceException(traceScope, fileSystemOperationUnhandledException);
+ throw new IOException(fileSystemOperationUnhandledException);
+ } finally {
+ tracingService.traceEnd(traceScope);
+ }
+ }
+
+ private void checkException(final Path path,
+ final AzureBlobFileSystemException exception,
+ final AzureServiceErrorCode... allowedErrorCodesList) throws IOException {
+ if (exception instanceof AbfsRestOperationException) {
+ AbfsRestOperationException ere = (AbfsRestOperationException) exception;
+
+ if (ArrayUtils.contains(allowedErrorCodesList, ere.getErrorCode())) {
+ return;
+ }
+ int statusCode = ere.getStatusCode();
+
+ //AbfsRestOperationException.getMessage() contains full error info including path/uri.
+ if (statusCode == HttpURLConnection.HTTP_NOT_FOUND) {
+ throw new FileNotFoundException(ere.getMessage());
+ } else if (statusCode == HttpURLConnection.HTTP_CONFLICT) {
+ throw new FileAlreadyExistsException(ere.getMessage());
+ } else {
+ throw ere;
+ }
+ } else {
+ if (path == null) {
+ throw exception;
+ }
+ // record info of path
+ throw new PathIOException(path.toString(), exception);
+ }
+ }
+
+ @VisibleForTesting
+ FileSystem.Statistics getFsStatistics() {
+ return this.statistics;
+ }
+
+ @VisibleForTesting
+ static class FileSystemOperation<T> {
+ private final T result;
+ private final AbfsRestOperationException exception;
+
+ FileSystemOperation(final T result, final AbfsRestOperationException exception) {
+ this.result = result;
+ this.exception = exception;
+ }
+
+ public boolean failed() {
+ return this.exception != null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/SecureAzureBlobFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/SecureAzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/SecureAzureBlobFileSystem.java
new file mode 100644
index 0000000..a6ad829
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/SecureAzureBlobFileSystem.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+
+/**
+ * A secure {@link org.apache.hadoop.fs.FileSystem} for reading and writing files stored on <a
+ * href="http://store.azure.com/">Windows Azure</a>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class SecureAzureBlobFileSystem extends AzureBlobFileSystem {
+ @Override
+ public boolean isSecure() {
+ return true;
+ }
+
+ @Override
+ public String getScheme() {
+ return FileSystemUriSchemes.ABFS_SECURE_SCHEME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java
new file mode 100644
index 0000000..2ec4db0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all constant keys used in abfs rest client here
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class AbfsHttpConstants {
+ // Abfs Http client constants
+ public static final String FILESYSTEM = "filesystem";
+ public static final String FILE = "file";
+ public static final String DIRECTORY = "directory";
+ public static final String APPEND_ACTION = "append";
+ public static final String FLUSH_ACTION = "flush";
+ public static final String SET_PROPERTIES_ACTION = "setProperties";
+ public static final String DEFAULT_TIMEOUT = "90";
+
+ public static final String JAVA_VERSION = "java.version";
+ public static final String OS_NAME = "os.name";
+ public static final String OS_VERSION = "os.version";
+
+ public static final String CLIENT_VERSION = "Azure Blob FS/1.0";
+
+ // Abfs Http Verb
+ public static final String HTTP_METHOD_DELETE = "DELETE";
+ public static final String HTTP_METHOD_GET = "GET";
+ public static final String HTTP_METHOD_HEAD = "HEAD";
+ public static final String HTTP_METHOD_PATCH = "PATCH";
+ public static final String HTTP_METHOD_POST = "POST";
+ public static final String HTTP_METHOD_PUT = "PUT";
+
+ // Abfs generic constants
+ public static final String SINGLE_WHITE_SPACE = " ";
+ public static final String EMPTY_STRING = "";
+ public static final String FORWARD_SLASH = "/";
+ public static final String DOT = ".";
+ public static final String PLUS = "+";
+ public static final String STAR = "*";
+ public static final String COMMA = ",";
+ public static final String EQUAL = "=";
+ public static final String QUESTION_MARK = "?";
+ public static final String AND_MARK = "&";
+ public static final String SEMICOLON = ";";
+ public static final String HTTP_HEADER_PREFIX = "x-ms-";
+
+ public static final String PLUS_ENCODE = "%20";
+ public static final String FORWARD_SLASH_ENCODE = "%2F";
+ public static final String AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER = "@";
+ public static final String UTF_8 = "utf-8";
+ public static final String GMT_TIMEZONE = "GMT";
+ public static final String APPLICATION_JSON = "application/json";
+ public static final String APPLICATION_OCTET_STREAM = "application/octet-stream";
+
+ private AbfsHttpConstants() {}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
new file mode 100644
index 0000000..ead1003
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class ConfigurationKeys {
+ public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+ public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)";
+ public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode";
+
+ // Retry strategy defined by the user
+ public static final String AZURE_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval";
+ public static final String AZURE_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval";
+ public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval";
+ public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries";
+
+ // Remove this and use common azure storage emulator property for public release.
+ public static final String FS_AZURE_EMULATOR_ENABLED = "fs.azure.abfs.emulator.enabled";
+
+ // Read and write buffer sizes defined by the user
+ public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size";
+ public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size";
+ public static final String AZURE_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size";
+ public static final String AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME = "fs.azure.block.location.impersonatedhost";
+ public static final String AZURE_CONCURRENT_CONNECTION_VALUE_OUT = "fs.azure.concurrentRequestCount.out";
+ public static final String AZURE_CONCURRENT_CONNECTION_VALUE_IN = "fs.azure.concurrentRequestCount.in";
+ public static final String AZURE_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append";
+ public static final String AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION = "fs.azure.createRemoteFileSystemDuringInitialization";
+ public static final String FS_AZURE_AUTOTHROTTLING_ENABLE = "fs.azure.autothrottling.enable";
+ public static final String FS_AZURE_ATOMIC_RENAME_KEY = "fs.azure.atomic.rename.key";
+ public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH = "fs.azure.readaheadqueue.depth";
+
+ private ConfigurationKeys() {}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
new file mode 100644
index 0000000..482158c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all the Azure Blob File System related configurations.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class FileSystemConfigurations {
+ public static final String USER_HOME_DIRECTORY_PREFIX = "/user";
+
+ // Retry parameter defaults.
+ public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s
+ public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s
+ public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s
+ public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
+
+ private static final int ONE_KB = 1024;
+ private static final int ONE_MB = ONE_KB * ONE_KB;
+
+ // Default upload and download buffer size
+ public static final int DEFAULT_WRITE_BUFFER_SIZE = 4 * ONE_MB; // 4 MB
+ public static final int DEFAULT_READ_BUFFER_SIZE = 4 * ONE_MB; // 4 MB
+ public static final int MIN_BUFFER_SIZE = 16 * ONE_KB; // 16 KB
+ public static final int MAX_BUFFER_SIZE = 100 * ONE_MB; // 100 MB
+ public static final long MAX_AZURE_BLOCK_SIZE = 512 * 1024 * 1024L;
+ public static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost";
+
+ public static final int MAX_CONCURRENT_READ_THREADS = 12;
+ public static final int MAX_CONCURRENT_WRITE_THREADS = 8;
+ public static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false;
+ public static final boolean DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION = false;
+
+ public static final String DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES = "/hbase";
+
+ public static final int DEFAULT_READ_AHEAD_QUEUE_DEPTH = -1;
+
+ private FileSystemConfigurations() {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java
new file mode 100644
index 0000000..c7a0cda
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all Azure Blob File System valid URI schemes.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class FileSystemUriSchemes {
+ public static final String ABFS_SCHEME = "abfs";
+ public static final String ABFS_SECURE_SCHEME = "abfss";
+ public static final String ABFS_DNS_PREFIX = "dfs";
+
+ public static final String HTTP_SCHEME = "http";
+ public static final String HTTPS_SCHEME = "https";
+
+ public static final String WASB_SCHEME = "wasb";
+ public static final String WASB_SECURE_SCHEME = "wasbs";
+ public static final String WASB_DNS_PREFIX = "blob";
+
+ private FileSystemUriSchemes() {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
new file mode 100644
index 0000000..9b7f9bc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all abfs http headers here
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class HttpHeaderConfigurations {
+ public static final String ACCEPT = "Accept";
+ public static final String ACCEPT_CHARSET = "Accept-Charset";
+ public static final String AUTHORIZATION = "Authorization";
+ public static final String IF_MODIFIED_SINCE = "If-Modified-Since";
+ public static final String IF_UNMODIFIED_SINCE = "If-Unmodified-Since";
+ public static final String IF_MATCH = "If-Match";
+ public static final String IF_NONE_MATCH = "If-None-Match";
+ public static final String CONTENT_LENGTH = "Content-Length";
+ public static final String CONTENT_ENCODING = "Content-Encoding";
+ public static final String CONTENT_LANGUAGE = "Content-Language";
+ public static final String CONTENT_MD5 = "Content-MD5";
+ public static final String CONTENT_TYPE = "Content-Type";
+ public static final String RANGE = "Range";
+ public static final String TRANSFER_ENCODING = "Transfer-Encoding";
+ public static final String USER_AGENT = "User-Agent";
+ public static final String X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override";
+ public static final String X_MS_CLIENT_REQUEST_ID = "x-ms-client-request-id";
+ public static final String X_MS_DATE = "x-ms-date";
+ public static final String X_MS_REQUEST_ID = "x-ms-request-id";
+ public static final String X_MS_VERSION = "x-ms-version";
+ public static final String X_MS_RESOURCE_TYPE = "x-ms-resource-type";
+ public static final String X_MS_CONTINUATION = "x-ms-continuation";
+ public static final String ETAG = "ETag";
+ public static final String X_MS_PROPERTIES = "x-ms-properties";
+ public static final String X_MS_RENAME_SOURCE = "x-ms-rename-source";
+ public static final String LAST_MODIFIED = "Last-Modified";
+
+ private HttpHeaderConfigurations() {}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java
new file mode 100644
index 0000000..a9f7d33
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all Http Query params here
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class HttpQueryParams {
+ public static final String QUERY_PARAM_RESOURCE = "resource";
+ public static final String QUERY_PARAM_DIRECTORY = "directory";
+ public static final String QUERY_PARAM_CONTINUATION = "continuation";
+ public static final String QUERY_PARAM_RECURSIVE = "recursive";
+ public static final String QUERY_PARAM_MAXRESULTS = "maxResults";
+ public static final String QUERY_PARAM_ACTION = "action";
+ public static final String QUERY_PARAM_POSITION = "position";
+ public static final String QUERY_PARAM_TIMEOUT = "timeout";
+ public static final String QUERY_PARAM_RETAIN_UNCOMMITTED_DATA = "retainUncommittedData";
+
+ private HttpQueryParams() {}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java
new file mode 100644
index 0000000..e6a471b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.azurebfs.constants;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java
new file mode 100644
index 0000000..462ebbc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contracts.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Definitions of Annotations for all types of the validators
+ */
+@InterfaceStability.Evolving
+public class ConfigurationValidationAnnotations {
+ /**
+ * Describes the requirements when validating the annotated int field
+ */
+ @Target({ ElementType.FIELD })
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface IntegerConfigurationValidatorAnnotation {
+ String ConfigurationKey();
+
+ int MaxValue() default Integer.MAX_VALUE;
+
+ int MinValue() default Integer.MIN_VALUE;
+
+ int DefaultValue();
+
+ boolean ThrowIfInvalid() default false;
+ }
+
+ /**
+ * Describes the requirements when validating the annotated long field
+ */
+ @Target({ ElementType.FIELD })
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface LongConfigurationValidatorAnnotation {
+ String ConfigurationKey();
+
+ long MaxValue() default Long.MAX_VALUE;
+
+ long MinValue() default Long.MIN_VALUE;
+
+ long DefaultValue();
+
+ boolean ThrowIfInvalid() default false;
+ }
+
+ /**
+ * Describes the requirements when validating the annotated String field
+ */
+ @Target({ ElementType.FIELD })
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface StringConfigurationValidatorAnnotation {
+ String ConfigurationKey();
+
+ String DefaultValue();
+
+ boolean ThrowIfInvalid() default false;
+ }
+
+ /**
+ * Describes the requirements when validating the annotated String field
+ */
+ @Target({ ElementType.FIELD })
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Base64StringConfigurationValidatorAnnotation {
+ String ConfigurationKey();
+
+ String DefaultValue();
+
+ boolean ThrowIfInvalid() default false;
+ }
+
+ /**
+ * Describes the requirements when validating the annotated boolean field
+ */
+ @Target({ ElementType.FIELD })
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface BooleanConfigurationValidatorAnnotation {
+ String ConfigurationKey();
+
+ boolean DefaultValue();
+
+ boolean ThrowIfInvalid() default false;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/package-info.java
new file mode 100644
index 0000000..0fc4deb
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.azurebfs.contracts.annotations;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/ConfigurationValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/ConfigurationValidator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/ConfigurationValidator.java
new file mode 100644
index 0000000..796f785
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/ConfigurationValidator.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contracts.diagnostics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
+
+/**
+ * ConfigurationValidator to validate the value of a configuration key
+ * @param <T> the type of the validator and the validated value
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ConfigurationValidator<T> {
+ /**
+ * Validates the configValue
+ * @return validated value of type T
+ */
+ T validate(String configValue) throws InvalidConfigurationValueException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/package-info.java
new file mode 100644
index 0000000..f8d27b2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/diagnostics/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.azurebfs.contracts.diagnostics;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java
new file mode 100644
index 0000000..f0b69ef
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.fs.azurebfs.contracts.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
+
+/**
+ * Exception to wrap Azure service error responses.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class AbfsRestOperationException extends AzureBlobFileSystemException {
+ private final int statusCode;
+ private final AzureServiceErrorCode errorCode;
+ private final String errorMessage;
+
+ public AbfsRestOperationException(
+ final int statusCode,
+ final String errorCode,
+ final String errorMessage,
+ final Exception innerException) {
+ super("Status code: " + statusCode + " error code: " + errorCode + " error message: " + errorMessage, innerException);
+
+ this.statusCode = statusCode;
+ this.errorCode = AzureServiceErrorCode.getAzureServiceCode(this.statusCode, errorCode);
+ this.errorMessage = errorMessage;
+ }
+
+ public AbfsRestOperationException(
+ final int statusCode,
+ final String errorCode,
+ final String errorMessage,
+ final Exception innerException,
+ final AbfsHttpOperation abfsHttpOperation) {
+ super(formatMessage(abfsHttpOperation));
+
+ this.statusCode = statusCode;
+ this.errorCode = AzureServiceErrorCode.getAzureServiceCode(this.statusCode, errorCode);
+ this.errorMessage = errorMessage;
+ }
+
+ public int getStatusCode() {
+ return this.statusCode;
+ }
+
+ public AzureServiceErrorCode getErrorCode() {
+ return this.errorCode;
+ }
+
+ public String getErrorMessage() {
+ return this.errorMessage;
+ }
+
+ private static String formatMessage(final AbfsHttpOperation abfsHttpOperation) {
+ return String.format(
+ "%1$s %2$s%nStatusCode=%3$s%nStatusDescription=%4$s%nErrorCode=%5$s%nErrorMessage=%6$s",
+ abfsHttpOperation.getMethod(),
+ abfsHttpOperation.getUrl().toString(),
+ abfsHttpOperation.getStatusCode(),
+ abfsHttpOperation.getStatusDescription(),
+ abfsHttpOperation.getStorageErrorCode(),
+ abfsHttpOperation.getStorageErrorMessage());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java
new file mode 100644
index 0000000..f31c680
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contracts.exceptions;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Base exception for any Azure Blob File System driver exceptions. All the exceptions must inherit this class.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class AzureBlobFileSystemException extends IOException {
+ public AzureBlobFileSystemException(final String message) {
+ super(message);
+ }
+
+ public AzureBlobFileSystemException(final String message, final Exception innerException) {
+ super(message, innerException);
+ }
+
+ @Override
+ public String toString() {
+ if (this.getMessage() == null && this.getCause() == null) {
+ return "";
+ }
+
+ if (this.getCause() == null) {
+ return this.getMessage();
+ }
+
+ if (this.getMessage() == null) {
+ return this.getCause().toString();
+ }
+
+ return this.getMessage() + this.getCause().toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java
new file mode 100644
index 0000000..bf3b2f3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.contracts.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown when a searched for element is not found
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ConfigurationPropertyNotFoundException extends AzureBlobFileSystemException {
+ public ConfigurationPropertyNotFoundException(String property) {
+ super("Configuration property " + property + "not found.");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org