You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2026/02/24 18:25:47 UTC
(hadoop) branch branch-3.5.0 updated: HADOOP-19778. [ABFS] Removing Deprecated Wasb Code
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a commit to branch branch-3.5.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.5.0 by this push:
new 32a391c4e2e HADOOP-19778. [ABFS] Removing Deprecated Wasb Code
32a391c4e2e is described below
commit 32a391c4e2e8f89bd3f1d8d8db05868e7e394ea0
Author: Anuj Modi <an...@apache.org>
AuthorDate: Tue Feb 24 17:14:09 2026 +0000
HADOOP-19778. [ABFS] Removing Deprecated Wasb Code
Closes #8207
Signed-off-by: Chris Nauroth <cn...@apache.org>
Reviewed-by: Anmol Asrani <an...@gmail.com>
Reviewed-by: manika137 <80...@users.noreply.github.com>
---
hadoop-tools/hadoop-azure/pom.xml | 245 +-
.../org/apache/hadoop/fs/azure/AzureException.java | 44 -
.../azure/AzureFileSystemThreadPoolExecutor.java | 347 --
.../apache/hadoop/fs/azure/AzureLinkedStack.java | 217 --
.../fs/azure/AzureNativeFileSystemStore.java | 3105 ---------------
.../hadoop/fs/azure/BlobMaterialization.java | 42 -
.../hadoop/fs/azure/BlobOperationDescriptor.java | 222 --
.../hadoop/fs/azure/BlockBlobAppendStream.java | 1141 ------
.../hadoop/fs/azure/BlockBlobInputStream.java | 482 ---
.../apache/hadoop/fs/azure/CachingAuthorizer.java | 232 --
.../hadoop/fs/azure/ClientThrottlingAnalyzer.java | 284 --
.../hadoop/fs/azure/ClientThrottlingIntercept.java | 221 --
.../org/apache/hadoop/fs/azure/FileMetadata.java | 125 -
.../org/apache/hadoop/fs/azure/KeyProvider.java | 44 -
.../hadoop/fs/azure/KeyProviderException.java | 42 -
.../hadoop/fs/azure/LocalSASKeyGeneratorImpl.java | 295 --
.../hadoop/fs/azure/NativeAzureFileSystem.java | 4058 +-------------------
.../fs/azure/NativeAzureFileSystemHelper.java | 155 -
.../hadoop/fs/azure/NativeFileSystemStore.java | 148 -
.../hadoop/fs/azure/PageBlobFormatHelpers.java | 58 -
.../hadoop/fs/azure/PageBlobInputStream.java | 506 ---
.../hadoop/fs/azure/PageBlobOutputStream.java | 607 ---
.../hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java | 337 --
.../hadoop/fs/azure/RemoteWasbAuthorizerImpl.java | 250 --
.../hadoop/fs/azure/SASKeyGenerationException.java | 40 -
.../hadoop/fs/azure/SASKeyGeneratorImpl.java | 61 -
.../hadoop/fs/azure/SASKeyGeneratorInterface.java | 66 -
.../hadoop/fs/azure/SecureModeException.java | 40 -
.../fs/azure/SecureStorageInterfaceImpl.java | 606 ---
.../fs/azure/SecureWasbRemoteCallHelper.java | 230 --
.../apache/hadoop/fs/azure/SelfRenewingLease.java | 213 -
.../hadoop/fs/azure/SelfThrottlingIntercept.java | 195 -
.../hadoop/fs/azure/SendRequestIntercept.java | 92 -
.../fs/azure/ShellDecryptionKeyProvider.java | 62 -
.../apache/hadoop/fs/azure/SimpleKeyProvider.java | 61 -
.../apache/hadoop/fs/azure/StorageInterface.java | 793 ----
.../hadoop/fs/azure/StorageInterfaceImpl.java | 522 ---
.../hadoop/fs/azure/SyncableDataOutputStream.java | 106 -
.../main/java/org/apache/hadoop/fs/azure/Wasb.java | 47 -
.../fs/azure/WasbAuthorizationException.java | 40 -
.../fs/azure/WasbAuthorizationOperations.java | 42 -
.../hadoop/fs/azure/WasbAuthorizerInterface.java | 53 -
.../java/org/apache/hadoop/fs/azure/WasbFsck.java | 200 -
.../hadoop/fs/azure/WasbRemoteCallException.java | 41 -
.../hadoop/fs/azure/WasbRemoteCallHelper.java | 297 --
.../java/org/apache/hadoop/fs/azure/Wasbs.java | 47 -
.../metrics/AzureFileSystemInstrumentation.java | 396 --
.../metrics/AzureFileSystemMetricsSystem.java | 69 -
.../fs/azure/metrics/BandwidthGaugeUpdater.java | 286 --
.../fs/azure/metrics/ErrorMetricUpdater.java | 82 -
.../metrics/ResponseReceivedMetricUpdater.java | 142 -
.../fs/azure/metrics/RollingWindowAverage.java | 103 -
.../apache/hadoop/fs/azure/metrics/package.html | 28 -
.../fs/azure/{security => }/package-info.java | 9 +-
.../java/org/apache/hadoop/fs/azure/package.html | 31 -
.../apache/hadoop/fs/azure/security/Constants.java | 50 -
.../apache/hadoop/fs/azure/security/JsonUtils.java | 52 -
.../security/RemoteWasbDelegationTokenManager.java | 169 -
.../hadoop/fs/azure/security/SpnegoToken.java | 49 -
.../hadoop/fs/azure/security/TokenUtils.java | 60 -
.../security/WasbDelegationTokenIdentifier.java | 48 -
.../azure/security/WasbDelegationTokenManager.java | 54 -
.../hadoop/fs/azure/security/WasbTokenRenewer.java | 91 -
...rg.apache.hadoop.security.token.TokenIdentifier | 1 -
.../org.apache.hadoop.security.token.TokenRenewer | 1 -
.../src/site/markdown/deprecated_wasb.md | 564 ---
.../hadoop-azure/src/site/markdown/fns_blob.md | 12 +-
.../hadoop-azure/src/site/markdown/index.md | 6 +-
.../src/site/markdown/testing_azure.md | 174 +-
.../hadoop-azure/src/site/markdown/wasb.md | 30 +-
.../src/site/markdown/wasbToAbfsMigration.md | 8 +-
.../hadoop/fs/azure/AbstractWasbTestBase.java | 175 -
.../fs/azure/AbstractWasbTestWithTimeout.java | 78 -
.../fs/azure/AzureBlobStorageTestAccount.java | 947 -----
.../fs/azure/ExceptionHandlingTestHelper.java | 67 -
.../fs/azure/ITestAzureConcurrentOutOfBandIo.java | 180 -
...stAzureConcurrentOutOfBandIoWithSecureMode.java | 33 -
.../azure/ITestAzureFileSystemErrorConditions.java | 242 --
.../hadoop/fs/azure/ITestBlobDataValidation.java | 242 --
.../fs/azure/ITestBlobTypeSpeedDifference.java | 163 -
.../hadoop/fs/azure/ITestBlockBlobInputStream.java | 927 -----
.../hadoop/fs/azure/ITestContainerChecks.java | 190 -
.../ITestFileSystemOperationExceptionHandling.java | 319 --
.../ITestFileSystemOperationExceptionMessage.java | 80 -
...emOperationsExceptionHandlingMultiThreaded.java | 381 --
.../ITestFileSystemOperationsWithThreads.java | 812 ----
.../hadoop/fs/azure/ITestListPerformance.java | 198 -
...ITestNativeAzureFSAuthWithBlobSpecificKeys.java | 40 -
.../ITestNativeAzureFSAuthorizationCaching.java | 53 -
.../fs/azure/ITestNativeAzureFSPageBlobLive.java | 43 -
.../fs/azure/ITestNativeAzureFileSystemAppend.java | 354 --
...stNativeAzureFileSystemAtomicRenameDirList.java | 55 -
.../ITestNativeAzureFileSystemClientLogging.java | 136 -
.../ITestNativeAzureFileSystemConcurrencyLive.java | 241 --
...ITestNativeAzureFileSystemContractEmulator.java | 67 -
.../ITestNativeAzureFileSystemContractLive.java | 107 -
...tNativeAzureFileSystemContractPageBlobLive.java | 115 -
.../fs/azure/ITestNativeAzureFileSystemLive.java | 309 --
.../fs/azure/ITestNativeFileSystemStatistics.java | 98 -
.../ITestOutOfBandAzureBlobOperationsLive.java | 185 -
.../fs/azure/ITestOutputStreamSemantics.java | 424 --
.../hadoop/fs/azure/ITestPageBlobInputStream.java | 523 ---
.../hadoop/fs/azure/ITestPageBlobOutputStream.java | 83 -
.../azure/ITestReadAndSeekPageBlobAfterWrite.java | 342 --
.../hadoop/fs/azure/ITestWasbRemoteCallHelper.java | 517 ---
.../fs/azure/ITestWasbUriAndConfiguration.java | 664 ----
.../hadoop/fs/azure/InMemoryBlockBlobStore.java | 197 -
.../hadoop/fs/azure/MockStorageInterface.java | 661 ----
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 215 --
.../fs/azure/NativeAzureFileSystemBaseTest.java | 1731 ---------
.../apache/hadoop/fs/azure/TestBlobMetadata.java | 269 --
.../fs/azure/TestBlobOperationDescriptor.java | 303 --
.../fs/azure/TestClientThrottlingAnalyzer.java | 167 -
.../fs/azure/TestKeyPageBlobDirectories.java | 169 -
.../TestNativeAzureFileSystemAuthorization.java | 2113 ----------
.../TestNativeAzureFileSystemBlockCompaction.java | 265 --
.../TestNativeAzureFileSystemConcurrency.java | 181 -
.../TestNativeAzureFileSystemContractMocked.java | 66 -
.../TestNativeAzureFileSystemFileNameCheck.java | 131 -
.../fs/azure/TestNativeAzureFileSystemMocked.java | 67 -
.../TestNativeAzureFileSystemOperationsMocked.java | 89 -
.../TestNativeAzureFileSystemUploadLogic.java | 181 -
.../fs/azure/TestOutOfBandAzureBlobOperations.java | 170 -
.../fs/azure/TestShellDecryptionKeyProvider.java | 81 -
.../fs/azure/TestSyncableDataOutputStream.java | 68 -
.../org/apache/hadoop/fs/azure/TestWasbFsck.java | 134 -
.../hadoop/fs/azure/TestWasbInitFailure.java | 71 +
.../contract/ITestAzureNativeContractAppend.java | 41 -
.../contract/ITestAzureNativeContractCreate.java | 34 -
.../contract/ITestAzureNativeContractDelete.java | 33 -
.../contract/ITestAzureNativeContractDistCp.java | 49 -
.../ITestAzureNativeContractGetFileStatus.java | 35 -
.../contract/ITestAzureNativeContractMkdir.java | 33 -
.../contract/ITestAzureNativeContractOpen.java | 34 -
.../contract/ITestAzureNativeContractRename.java | 34 -
.../contract/ITestAzureNativeContractSeek.java | 34 -
.../contract/NativeAzureFileSystemContract.java | 49 -
.../azure/integration/AbstractAzureScaleTest.java | 71 -
.../fs/azure/integration/AzureTestConstants.java | 191 -
.../fs/azure/integration/AzureTestUtils.java | 557 ---
.../azure/integration/CleanupTestContainers.java | 87 -
.../fs/azure/integration/ITestAzureHugeFiles.java | 459 ---
.../apache/hadoop/fs/azure/integration/Sizes.java | 43 -
.../fs/azure/metrics/AzureMetricsTestUtil.java | 83 -
.../ITestAzureFileSystemInstrumentation.java | 546 ---
.../azure/metrics/TestBandwidthGaugeUpdater.java | 78 -
.../TestNativeAzureFileSystemMetricsSystem.java | 100 -
.../fs/azure/metrics/TestRollingWindowAverage.java | 42 -
.../org/apache/hadoop/fs/azure/package-info.java} | 14 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 45 -
.../hadoop/fs/azurebfs/AbstractAbfsScaleTest.java | 12 +-
.../hadoop/fs/azurebfs/ITestAbfsHugeFiles.java | 10 +-
.../ITestAzureBlobFileSystemRandomRead.java | 103 -
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2044 ----------
.../azurebfs/commit/AbstractAbfsClusterITest.java | 6 +-
.../commit/ITestAbfsLoadManifestsStage.java | 5 +-
.../azurebfs/constants/TestConfigurationKeys.java | 3 +
.../ITestAbfsFileSystemContractDistCp.java | 6 +-
.../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 194 +-
.../hadoop-azure/src/test/resources/wasb.xml | 169 -
160 files changed, 402 insertions(+), 41137 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 4809ebf66c4..123b58ed528 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -26,7 +26,7 @@
<name>Apache Hadoop Azure support</name>
<description>
This module contains code to support integration with Azure.
- Currently this consists of a filesystem client to read data from
+ Currently, this consists of Azure Blob File System client to read data from
and write data to Azure Storage.
</description>
<packaging>jar</packaging>
@@ -65,6 +65,7 @@
<effort>Max</effort>
</configuration>
</plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -89,7 +90,7 @@
</executions>
</plugin>
- <plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
@@ -109,7 +110,6 @@
</plugins>
-
<!--
The following is to suppress a m2e warning in eclipse
(m2e doesn't know how to handle maven-enforcer:enforce, so we have to tell m2e to ignore it)
@@ -153,7 +153,6 @@
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
-
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -244,8 +243,8 @@
except when using this committer, so it only tagged as
"provided".
It is not exported as a transitive dependency of the JAR.
- Applications which wish to to use the manifest committer
- will need to explicity add the mapreduce JAR to their classpath.
+ Applications which wish to use the manifest committer
+ will need to explicitly add the mapreduce JAR to their classpath.
This is already done by MapReduce and Spark.
-->
<dependency>
@@ -253,6 +252,7 @@
<artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>
+
<!--
These are only added to the classpath for tests,
and are not exported by the test JAR as transitive
@@ -264,41 +264,48 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
<scope>test</scope>
<type>jar</type>
</dependency>
+
<!-- artifacts needed to bring up a Mini MR Yarn cluster-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
@@ -329,37 +336,44 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<scope>test</scope>
</dependency>
+
<!-- Used to create SSL certs for a secure Keystore -->
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk18on</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
@@ -368,169 +382,6 @@
</dependencies>
<profiles>
- <profile>
- <id>parallel-tests-wasb</id>
- <activation>
- <property>
- <name>parallel-tests</name>
- <value>wasb</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>parallel-tests-createdir</id>
- <goals>
- <goal>parallel-tests-createdir</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <executions>
- <execution>
- <id>default-test</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <forkCount>${testsThreadCount}</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- <!-- http connection pool size -->
- <http.maxConnections>${http.maxConnections}</http.maxConnections> </systemPropertyVariables>
- <includes>
- <include>**/azure/Test*.java</include>
- <include>**/azure/**/Test*.java</include>
- </includes>
- <excludes>
- <exclude>**/azure/**/TestRollingWindowAverage*.java</exclude>
- </excludes>
- </configuration>
- </execution>
- <execution>
- <id>serialized-test-wasb</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <forkCount>1</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- <!-- http connection pool size -->
- <http.maxConnections>${http.maxConnections}</http.maxConnections>
- </systemPropertyVariables>
- <includes>
- <include>**/azure/**/TestRollingWindowAverage*.java</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <executions>
- <execution>
- <id>default-integration-test-wasb</id>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- <configuration>
- <forkCount>${testsThreadCount}</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <!-- Tell tests that they are being executed in parallel -->
- <test.parallel.execution>true</test.parallel.execution>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-
- <!-- Due to a Maven quirk, setting this to just -->
- <!-- surefire.forkNumber won't do the parameter -->
- <!-- substitution. Putting a prefix in front of it like -->
- <!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <!-- Propagate scale parameters -->
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- </systemPropertyVariables>
- <!-- Some tests cannot run in parallel-->
- <includes>
- <include>**/azure/ITest*.java</include>
- <include>**/azure/**/ITest*.java</include>
- </includes>
- <excludes>
- <exclude>**/azure/ITestNativeFileSystemStatistics.java</exclude>
- </excludes>
- </configuration>
- </execution>
- <!-- Do a sequential run for tests that cannot handle -->
- <!-- parallel execution. -->
- <execution>
- <id>sequential-integration-tests-wasb</id>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- <configuration>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.parallel.execution>false</test.parallel.execution>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- </systemPropertyVariables>
- <includes>
- <include>**/azure/ITestNativeFileSystemStatistics.java</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
<profile>
<id>parallel-tests-abfs</id>
@@ -764,36 +615,6 @@
<includes>
<include>**/Test*.java</include>
</includes>
- <excludes>
- <exclude>**/TestRollingWindowAverage*.java</exclude>
- </excludes>
- </configuration>
- </execution>
- <execution>
- <id>serialized-test</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <forkCount>1</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- </systemPropertyVariables>
- <includes>
- <include>**/TestRollingWindowAverage*.java</include>
- </includes>
</configuration>
</execution>
</executions>
@@ -850,18 +671,7 @@
<include>**/ITest*.java</include>
</includes>
<excludes>
- <exclude>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</exclude>
- <exclude>**/ITestFileSystemOperationsWithThreads.java</exclude>
- <exclude>**/ITestOutOfBandAzureBlobOperationsLive.java</exclude>
- <exclude>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</exclude>
- <exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude>
- <exclude>**/ITestNativeAzureFileSystemLive.java</exclude>
- <exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude>
<exclude>**/ITestAzureBlobFileSystemRandomRead.java</exclude>
- <exclude>**/ITestWasbRemoteCallHelper.java</exclude>
- <exclude>**/ITestBlockBlobInputStream.java</exclude>
- <exclude>**/ITestWasbAbfsCompatibility.java</exclude>
- <exclude>**/ITestNativeFileSystemStatistics.java</exclude>
</excludes>
</configuration>
</execution>
@@ -884,20 +694,10 @@
<fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
<fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
<!-- http connection pool size -->
- <http.maxConnections>${http.maxConnections}</http.maxConnections> </systemPropertyVariables>
+ <http.maxConnections>${http.maxConnections}</http.maxConnections>
+ </systemPropertyVariables>
<includes>
- <include>**/ITestWasbAbfsCompatibility.java</include>
- <include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
- <include>**/ITestFileSystemOperationsWithThreads.java</include>
- <include>**/ITestOutOfBandAzureBlobOperationsLive.java</include>
- <include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include>
- <include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include>
- <include>**/ITestNativeAzureFileSystemLive.java</include>
- <include>**/ITestNativeAzureFSPageBlobLive.java</include>
<include>**/ITestAzureBlobFileSystemRandomRead.java</include>
- <include>**/ITestWasbRemoteCallHelper.java</include>
- <include>**/ITestBlockBlobInputStream.java</include>
- <include>**/ITestNativeFileSystemStatistics.java</include>
</includes>
</configuration>
</execution>
@@ -957,5 +757,6 @@
<fs.azure.scale.test.enabled>true</fs.azure.scale.test.enabled>
</properties>
</profile>
+
</profiles>
</project>
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java
deleted file mode 100644
index 48ec064d5fa..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Thrown if there is a problem communicating with Azure Storage service.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class AzureException extends IOException {
- private static final long serialVersionUID = 1L;
-
- public AzureException(String message) {
- super(message);
- }
-
- public AzureException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public AzureException(Throwable t) {
- super(t);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java
deleted file mode 100644
index 98d28d1ea42..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-
-class AzureFileSystemThreadPoolExecutor {
-
- public static final Logger LOG = LoggerFactory.getLogger(AzureFileSystemThreadPoolExecutor.class);
-
- /*
- * Number of threads to keep in the pool.
- */
- private int threadCount;
-
- /*
- * Prefix to be used for naming threads.
- */
- private String threadNamePrefix;
-
- /*
- * File system operation like delete/rename. Used for logging purpose.
- */
- private String operation;
-
- /*
- * Source blob key used for file operation. Used for logging purpose.
- */
- private String key;
-
- /*
- * Configuration name for recommendations. Used for logging purpose.
- */
- private String config;
-
- /**
- * Creates a new AzureFileSystemThreadPoolExecutor object.
- *
- * @param threadCount
- * Number of threads to be used after reading user configuration.
- * @param threadNamePrefix
- * Prefix to be used to name threads for the file operation.
- * @param operation
- * File system operation like delete/rename. Used for logging purpose.
- * @param key
- * Source blob key used for file operation. Used for logging purpose.
- * @param config
- * Configuration name for recommendations. Used for logging purpose.
- */
- public AzureFileSystemThreadPoolExecutor(int threadCount, String threadNamePrefix,
- String operation, String key, String config) {
- this.threadCount = threadCount;
- this.threadNamePrefix = threadNamePrefix;
- this.operation = operation;
- this.key = key;
- this.config = config;
- }
-
- /**
- * Gets a new thread pool
- * @param threadCount
- * Number of threads to keep in the pool.
- * @param threadNamePrefix
- * Prefix to be used for naming threads.
- *
- * @return
- * Returns a new thread pool.
- */
- @VisibleForTesting
- ThreadPoolExecutor getThreadPool(int threadCount) throws Exception {
- return new ThreadPoolExecutor(threadCount, threadCount, 2, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new AzureFileSystemThreadFactory(this.threadNamePrefix));
- }
-
- /**
- * Execute the file operation parallel using threads. All threads works on a
- * single working set of files stored in input 'contents'. The synchronization
- * between multiple threads is achieved through retrieving atomic index value
- * from the array. Once thread gets the index, it retrieves the file and initiates
- * the file operation. The advantage with this method is that file operations
- * doesn't get serialized due to any thread. Also, the input copy is not changed
- * such that caller can reuse the list for other purposes.
- *
- * This implementation also considers that failure of operation on single file
- * is considered as overall operation failure. All threads bail out their execution
- * as soon as they detect any single thread either got exception or operation is failed.
- *
- * @param contents
- * List of blobs on which operation to be done.
- * @param threadOperation
- * The actual operation to be executed by each thread on a file.
- *
- * @param operationStatus
- * Returns true if the operation is success, false if operation is failed.
- * @throws IOException
- *
- */
- boolean executeParallel(FileMetadata[] contents, AzureFileSystemThreadTask threadOperation) throws IOException {
-
- boolean operationStatus = false;
- boolean threadsEnabled = false;
- int threadCount = this.threadCount;
- ThreadPoolExecutor ioThreadPool = null;
-
- // Start time for file operation
- long start = Time.monotonicNow();
-
- // If number of files are less then reduce threads to file count.
- threadCount = Math.min(contents.length, threadCount);
-
- if (threadCount > 1) {
- try {
- ioThreadPool = getThreadPool(threadCount);
- threadsEnabled = true;
- } catch(Exception e) {
- // The possibility of this scenario is very remote. Added this code as safety net.
- LOG.warn("Failed to create thread pool with threads {} for operation {} on blob {}."
- + " Use config {} to set less number of threads. Setting config value to <= 1 will disable threads.",
- threadCount, operation, key, config);
- }
- } else {
- LOG.warn("Disabling threads for {} operation as thread count {} is <= 1", operation, threadCount);
- }
-
- if (threadsEnabled) {
- LOG.debug("Using thread pool for {} operation with threads {}", operation, threadCount);
- boolean started = false;
- AzureFileSystemThreadRunnable runnable = new AzureFileSystemThreadRunnable(contents, threadOperation, operation);
-
- // Don't start any new requests if there is an exception from any one thread.
- for (int i = 0; i < threadCount && runnable.lastException == null && runnable.operationStatus; i++)
- {
- try {
- ioThreadPool.execute(runnable);
- started = true;
- } catch (RejectedExecutionException ex) {
- // If threads can't be scheduled then report error and move ahead with next thread.
- // Don't fail operation due to this issue.
- LOG.error("Rejected execution of thread for {} operation on blob {}."
- + " Continuing with existing threads. Use config {} to set less number of threads"
- + " to avoid this error", operation, key, config);
- }
- }
-
- // Stop accepting any new execute requests.
- ioThreadPool.shutdown();
-
- try {
- // Wait for threads to terminate. Keep time out as large value
- ioThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
- } catch(InterruptedException intrEx) {
- // If current thread got interrupted then shutdown all threads now.
- ioThreadPool.shutdownNow();
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- LOG.error("Threads got interrupted {} blob operation for {} "
- , operation, key);
- }
-
- int threadsNotUsed = threadCount - runnable.threadsUsed.get();
- if (threadsNotUsed > 0) {
- LOG.warn("{} threads not used for {} operation on blob {}", threadsNotUsed, operation, key);
- }
-
- if (!started) {
- // No threads started. Fall back to serial mode.
- threadsEnabled = false;
- LOG.info("Not able to schedule threads to {} blob {}. Fall back to {} blob serially."
- , operation, key, operation);
- } else {
- IOException lastException = runnable.lastException;
-
- // There are no exceptions from threads and no operation failures. Consider this scenario
- // as failure only if file operations are not done on all files.
- if (lastException == null && runnable.operationStatus && runnable.filesProcessed.get() < contents.length) {
- LOG.error("{} failed as operation on subfolders and files failed.", operation);
- lastException = new IOException(operation + " failed as operation on subfolders and files failed.");
- }
-
- if (lastException != null) {
- // Threads started and executed. One or more threads seems to have hit exception.
- // Raise the same exception.
- throw lastException;
- }
-
- operationStatus = runnable.operationStatus;
- }
- }
-
- if (!threadsEnabled) {
- // No threads. Serialize the operation. Clear any last exceptions.
- LOG.debug("Serializing the {} operation", operation);
- for (int i = 0; i < contents.length; i++) {
- if (!threadOperation.execute(contents[i])) {
- LOG.warn("Failed to {} file {}", operation, contents[i]);
- return false;
- }
- }
-
- // Operation is success
- operationStatus = true;
- }
-
- // Find the duration of time taken for file operation
- long end = Time.monotonicNow();
- LOG.info("Time taken for {} operation is: {} ms with threads: {}", operation, (end - start), threadCount);
-
- return operationStatus;
- }
-
- /**
- * A ThreadFactory for Azure File operation threads with meaningful names helpful
- * for debugging purposes.
- */
- static class AzureFileSystemThreadFactory implements ThreadFactory {
-
- private String threadIdPrefix = "AzureFileSystemThread";
-
- /**
- * Atomic integer to provide thread id for thread names.
- */
- private AtomicInteger threadSequenceNumber = new AtomicInteger(0);
-
- public AzureFileSystemThreadFactory(String prefix) {
- threadIdPrefix = prefix;
- }
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new SubjectInheritingThread(r);
-
- // Use current thread name as part in naming thread such that use of
- // same file system object will have unique names.
- t.setName(String.format("%s-%s-%d", threadIdPrefix, Thread.currentThread().getName(),
- threadSequenceNumber.getAndIncrement()));
- return t;
- }
-
- }
-
- static class AzureFileSystemThreadRunnable implements Runnable {
-
- // Tracks if any thread has raised exception.
- private volatile IOException lastException = null;
-
- // Tracks if any thread has failed execution.
- private volatile boolean operationStatus = true;
-
- // Atomic tracker to retrieve index of next file to be processed
- private AtomicInteger fileIndex = new AtomicInteger(0);
-
- // Atomic tracker to count number of files successfully processed
- private AtomicInteger filesProcessed = new AtomicInteger(0);
-
- // Atomic tracker to retrieve number of threads used to do at least one file operation.
- private AtomicInteger threadsUsed = new AtomicInteger(0);
-
- // Type of file system operation
- private String operation = "Unknown";
-
- // List of files to be processed.
- private final FileMetadata[] files;
-
- // Thread task which encapsulates the file system operation work on a file.
- private AzureFileSystemThreadTask task;
-
- public AzureFileSystemThreadRunnable(final FileMetadata[] files,
- AzureFileSystemThreadTask task, String operation) {
- this.operation = operation;
- this.files = files;
- this.task = task;
- }
-
- @Override
- public void run() {
- long start = Time.monotonicNow();
- int currentIndex;
- int processedFilesCount = 0;
-
- while ((currentIndex = fileIndex.getAndIncrement()) < files.length) {
- processedFilesCount++;
- FileMetadata file = files[currentIndex];
-
- try {
- // Execute the file operation.
- if (!task.execute(file)) {
- LOG.error("{} operation failed for file {}",
- this.operation, file.getKey());
- operationStatus = false;
- } else {
- filesProcessed.getAndIncrement();
- }
- } catch (Exception e) {
- LOG.error("Encountered Exception for {} operation for file {}",
- this.operation, file.getKey());
- lastException = new IOException("Encountered Exception for "
- + this.operation + " operation for file " + file.getKey(), e);
- }
-
- // If any thread has seen exception or operation failed then we
- // don't have to process further.
- if (lastException != null || !operationStatus) {
- LOG.warn("Terminating execution of {} operation now as some other thread"
- + " already got exception or operation failed", this.operation, file.getKey());
- break;
- }
- }
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to process {} files count for {} operation: {} ms",
- processedFilesCount, this.operation, (end - start));
- if (processedFilesCount > 0) {
- threadsUsed.getAndIncrement();
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java
deleted file mode 100644
index 4c52ef0e931..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * A simple generic stack implementation using linked lists. The stack
- * implementation has five main operations:
- * <ul>
- * <li>push -- adds an element to the top of the stack</li>
- * <li>pop -- removes an element from the top of the stack and returns a
- * reference to it</li>
- * <li>peek -- peek returns an element from the top of the stack without
- * removing it</li>
- * <li>isEmpty -- tests whether the stack is empty</li>
- * <li>size -- returns the size of the stack</li>
- * <li>toString -- returns a string representation of the stack.</li>
- * </ul>
- */
-
-public class AzureLinkedStack<E> {
- /*
- * Linked node for Azure stack collection.
- */
- private static class AzureLinkedNode<E> {
- private E element; // Linked element on the list.
- private AzureLinkedNode<E> next;// Reference to the next linked element on
- // list.
-
- /*
- * The constructor builds the linked node with no successor
- *
- * @param element : The value of the element to be stored with this node.
- */
- private AzureLinkedNode(E anElement) {
- element = anElement;
- next = null;
- }
-
- /*
- * Constructor builds a linked node with a specified successor. The
- * successor may be null.
- *
- * @param anElement : new element to be created.
- *
- * @param nextElement: successor to the new element.
- */
- private AzureLinkedNode(E anElement, AzureLinkedNode<E> nextElement) {
- element = anElement;
- next = nextElement;
- }
-
- /*
- * Get the element stored in the linked node.
- *
- * @return E : element stored in linked node.
- */
- private E getElement() {
- return element;
- }
-
- /*
- * Get the successor node to the element.
- *
- * @return E : reference to the succeeding node on the list.
- */
- private AzureLinkedNode<E> getNext() {
- return next;
- }
- }
-
- private int count; // The number of elements stored on the stack.
- private AzureLinkedNode<E> top; // Top of the stack.
-
- /*
- * Constructor creating an empty stack.
- */
- public AzureLinkedStack() {
- // Simply initialize the member variables.
- //
- count = 0;
- top = null;
- }
-
- /*
- * Adds an element to the top of the stack.
- *
- * @param element : element pushed to the top of the stack.
- */
- public void push(E element) {
- // Create a new node containing a reference to be placed on the stack.
- // Set the next reference to the new node to point to the current top
- // of the stack. Set the top reference to point to the new node. Finally
- // increment the count of nodes on the stack.
- //
- AzureLinkedNode<E> newNode = new AzureLinkedNode<E>(element, top);
- top = newNode;
- count++;
- }
-
- /*
- * Removes the element at the top of the stack and returns a reference to it.
- *
- * @return E : element popped from the top of the stack.
- *
- * @throws Exception on pop from an empty stack.
- */
- public E pop() throws Exception {
- // Make sure the stack is not empty. If it is empty, throw a StackEmpty
- // exception.
- //
- if (isEmpty()) {
- throw new Exception("AzureStackEmpty");
- }
-
- // Set a temporary reference equal to the element at the top of the stack,
- // decrement the count of elements and return reference to the temporary.
- //
- E element = top.getElement();
- top = top.getNext();
- count--;
-
- // Return the reference to the element that was at the top of the stack.
- //
- return element;
- }
-
- /*
- * Return the top element of the stack without removing it.
- *
- * @return E
- *
- * @throws Exception on peek into an empty stack.
- */
- public E peek() throws Exception {
- // Make sure the stack is not empty. If it is empty, throw a StackEmpty
- // exception.
- //
- if (isEmpty()) {
- throw new Exception("AzureStackEmpty");
- }
-
- // Set a temporary reference equal to the element at the top of the stack
- // and return the temporary.
- //
- E element = top.getElement();
- return element;
- }
-
- /*
- * Determines whether the stack is empty
- *
- * @return boolean true if the stack is empty and false otherwise.
- */
- public boolean isEmpty() {
- if (0 == size()) {
- // Zero-sized stack so the stack is empty.
- //
- return true;
- }
-
- // The stack is not empty.
- //
- return false;
- }
-
- /*
- * Determines the size of the stack
- *
- * @return int: Count of the number of elements in the stack.
- */
- public int size() {
- return count;
- }
-
- /*
- * Returns a string representation of the stack.
- *
- * @return String String representation of all elements in the stack.
- */
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
-
- AzureLinkedNode<E> current = top;
- for (int i = 0; i < size(); i++) {
- E element = current.getElement();
- sb.append(element.toString());
- current = current.getNext();
-
- // Insert commas between strings except after the last string.
- //
- if (size() - 1 > i) {
- sb.append(", ");
- }
- }
-
- // Return the string.
- //
- return sb.toString();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
deleted file mode 100644
index 6e50cfc627e..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ /dev/null
@@ -1,3105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.PATH_DELIMITER;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobContainerWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-import org.apache.hadoop.fs.azure.metrics.BandwidthGaugeUpdater;
-import org.apache.hadoop.fs.azure.metrics.ErrorMetricUpdater;
-import org.apache.hadoop.fs.azure.metrics.ResponseReceivedMetricUpdater;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.VersionInfo;
-import org.eclipse.jetty.util.ajax.JSON;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryExponentialRetry;
-import com.microsoft.azure.storage.RetryNoRetry;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
-import com.microsoft.azure.storage.StorageErrorCodeStrings;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.Constants;
-import com.microsoft.azure.storage.StorageEvent;
-import com.microsoft.azure.storage.core.BaseRequest;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.BlobProperties;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlobType;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CopyStatus;
-import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.core.Utility;
-
-/**
- * Core implementation of Windows Azure Filesystem for Hadoop.
- * Provides the bridging logic between Hadoop's abstract filesystem and Azure Storage
- *
- */
-@InterfaceAudience.Private
-@VisibleForTesting
-public class AzureNativeFileSystemStore implements NativeFileSystemStore {
-
- /**
- * Configuration knob on whether we do block-level MD5 validation on
- * upload/download.
- */
- static final String KEY_CHECK_BLOCK_MD5 = "fs.azure.check.block.md5";
- /**
- * Configuration knob on whether we store blob-level MD5 on upload.
- */
- static final String KEY_STORE_BLOB_MD5 = "fs.azure.store.blob.md5";
- static final String DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME = "storageemulator";
- static final String STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.storage.emulator.account.name";
-
- /**
- * Configuration for User-Agent field.
- */
- static final String USER_AGENT_ID_KEY = "fs.azure.user.agent.prefix";
- static final String USER_AGENT_ID_DEFAULT = "unknown";
-
- public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class);
-
- private StorageInterface storageInteractionLayer;
- private CloudBlobDirectoryWrapper rootDirectory;
- private CloudBlobContainerWrapper container;
-
- // Constants local to this class.
- //
- private static final String KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider.";
- private static final String KEY_ACCOUNT_SAS_PREFIX = "fs.azure.sas.";
-
- // note: this value is not present in core-default.xml as our real default is
- // computed as min(2*cpu,8)
- private static final String KEY_CONCURRENT_CONNECTION_VALUE_OUT = "fs.azure.concurrentRequestCount.out";
-
- private static final String HADOOP_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size";
- private static final String KEY_STREAM_MIN_READ_SIZE = "fs.azure.read.request.size";
- private static final String KEY_STORAGE_CONNECTION_TIMEOUT = "fs.azure.storage.timeout";
- private static final String KEY_WRITE_BLOCK_SIZE = "fs.azure.write.request.size";
- @VisibleForTesting
- static final String KEY_INPUT_STREAM_VERSION = "fs.azure.input.stream.version.for.internal.use.only";
-
- // Property controlling whether to allow reads on blob which are concurrently
- // appended out-of-band.
- private static final String KEY_READ_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append";
-
- // Configurable throttling parameter properties. These properties are located
- // in the core-site.xml configuration file.
- private static final String KEY_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval";
- private static final String KEY_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval";
- private static final String KEY_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval";
- private static final String KEY_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries";
-
- private static final String KEY_COPYBLOB_MIN_BACKOFF_INTERVAL =
- "fs.azure.io.copyblob.retry.min.backoff.interval";
- private static final String KEY_COPYBLOB_MAX_BACKOFF_INTERVAL =
- "fs.azure.io.copyblob.retry.max.backoff.interval";
- private static final String KEY_COPYBLOB_BACKOFF_INTERVAL =
- "fs.azure.io.copyblob.retry.backoff.interval";
- private static final String KEY_COPYBLOB_MAX_IO_RETRIES =
- "fs.azure.io.copyblob.retry.max.retries";
-
- private static final String KEY_SELF_THROTTLE_ENABLE = "fs.azure.selfthrottling.enable";
- private static final String KEY_SELF_THROTTLE_READ_FACTOR = "fs.azure.selfthrottling.read.factor";
- private static final String KEY_SELF_THROTTLE_WRITE_FACTOR = "fs.azure.selfthrottling.write.factor";
-
- private static final String KEY_AUTO_THROTTLE_ENABLE = "fs.azure.autothrottling.enable";
-
- private static final String KEY_ENABLE_STORAGE_CLIENT_LOGGING = "fs.azure.storage.client.logging";
-
- /**
- * Configuration keys to identify if WASB needs to run in Secure mode. In Secure mode
- * all interactions with Azure storage is performed using SAS uris. There are two sub modes
- * within the Secure mode , one is remote SAS key mode where the SAS keys are generated
- * from a remote process and local mode where SAS keys are generated within WASB.
- */
- @VisibleForTesting
- public static final String KEY_USE_SECURE_MODE = "fs.azure.secure.mode";
-
- /**
- * By default the SAS Key mode is expected to run in Romote key mode. This flags sets it
- * to run on the local mode.
- */
- public static final String KEY_USE_LOCAL_SAS_KEY_MODE = "fs.azure.local.sas.key.mode";
-
- /**
- * Config to control case sensitive metadata key checks/retrieval. If this
- * is false, blob metadata keys will be treated case insensitive.
- */
- private static final String KEY_BLOB_METADATA_KEY_CASE_SENSITIVE = "fs.azure.blob.metadata.key.case.sensitive";
- private static final String PERMISSION_METADATA_KEY = "hdi_permission";
- private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission";
- private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder";
- private static final String OLD_IS_FOLDER_METADATA_KEY = "asv_isfolder";
- static final String VERSION_METADATA_KEY = "hdi_version";
- static final String OLD_VERSION_METADATA_KEY = "asv_version";
- static final String FIRST_WASB_VERSION = "2013-01-01";
- static final String CURRENT_WASB_VERSION = "2013-09-01";
- static final String LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY = "hdi_tmpupload";
- static final String OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY = "asv_tmpupload";
-
- /**
- * Configuration key to indicate the set of directories in WASB where we
- * should store files as page blobs instead of block blobs.
- *
- * Entries should be plain directory names (i.e. not URIs) with no leading or
- * trailing slashes. Delimit the entries with commas.
- */
- public static final String KEY_PAGE_BLOB_DIRECTORIES =
- "fs.azure.page.blob.dir";
- /**
- * The set of directories where we should store files as page blobs.
- */
- private Set<String> pageBlobDirs;
-
- /**
- * Configuration key to indicate the set of directories in WASB where we
- * should store files as block blobs with block compaction enabled.
- *
- * Entries can be directory paths relative to the container (e.g. "/path") or
- * fully qualified wasb:// URIs (e.g.
- * wasb://container@example.blob.core.windows.net/path)
- */
- public static final String KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES =
- "fs.azure.block.blob.with.compaction.dir";
-
- /**
- * The set of directories where we should store files as block blobs with
- * block compaction enabled.
- */
- private Set<String> blockBlobWithCompationDirs;
-
- /**
- * Configuration key to indicate the set of directories in WASB where
- * we should do atomic folder rename synchronized with createNonRecursive.
- */
- public static final String KEY_ATOMIC_RENAME_DIRECTORIES =
- "fs.azure.atomic.rename.dir";
-
- /**
- * Configuration key to enable flat listing of blobs. This config is useful
- * only if listing depth is AZURE_UNBOUNDED_DEPTH.
- */
- public static final String KEY_ENABLE_FLAT_LISTING = "fs.azure.flatlist.enable";
-
- /**
- * Optional config to enable a lock free pread which will bypass buffer in
- * BlockBlobInputStream.
- * This is not a config which can be set at cluster level. It can be used as
- * an option on FutureDataInputStreamBuilder.
- * @see FileSystem#openFile(org.apache.hadoop.fs.Path)
- */
- public static final String FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE =
- "fs.azure.block.blob.buffered.pread.disable";
-
- /**
- * The set of directories where we should apply atomic folder rename
- * synchronized with createNonRecursive.
- */
- private Set<String> atomicRenameDirs;
-
- private static final String HTTP_SCHEME = "http";
- private static final String HTTPS_SCHEME = "https";
- private static final String WASB_AUTHORITY_DELIMITER = "@";
- private static final char ASTERISK_SYMBOL = '*';
- private static final String AZURE_ROOT_CONTAINER = "$root";
-
- private static final int DEFAULT_CONCURRENT_WRITES = 8;
-
- private static final Charset METADATA_ENCODING = StandardCharsets.UTF_8;
-
- // Concurrent reads reads of data written out of band are disable by default.
- //
- private static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false;
-
- // Default block sizes
- public static final int DEFAULT_DOWNLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
- public static final int DEFAULT_UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
- public static final long DEFAULT_HADOOP_BLOCK_SIZE = 512 * 1024 * 1024L;
-
- private static final int DEFAULT_INPUT_STREAM_VERSION = 2;
-
- // Retry parameter defaults.
- //
-
- private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s
- private static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s
- private static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s
- private static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
-
- private static final int DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL = 3 * 1000;
- private static final int DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL = 90 * 1000;
- private static final int DEFAULT_COPYBLOB_BACKOFF_INTERVAL = 30 * 1000;
- private static final int DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS = 15;
-
- // Self-throttling defaults. Allowed range = (0,1.0]
- // Value of 1.0 means no self-throttling.
- // Value of x means process data at factor x of unrestricted rate
- private static final boolean DEFAULT_SELF_THROTTLE_ENABLE = true;
- private static final float DEFAULT_SELF_THROTTLE_READ_FACTOR = 1.0f;
- private static final float DEFAULT_SELF_THROTTLE_WRITE_FACTOR = 1.0f;
-
- private static final boolean DEFAULT_AUTO_THROTTLE_ENABLE = false;
-
- private static final int STORAGE_CONNECTION_TIMEOUT_DEFAULT = 90;
-
- /**
- * Default values to control SAS Key mode.
- * By default we set the values to false.
- */
- public static final boolean DEFAULT_USE_SECURE_MODE = false;
- private static final boolean DEFAULT_USE_LOCAL_SAS_KEY_MODE = false;
-
- /**
- * Enable flat listing of blobs as default option. This is useful only if
- * listing depth is AZURE_UNBOUNDED_DEPTH.
- */
- public static final boolean DEFAULT_ENABLE_FLAT_LISTING = false;
-
- /**
- * MEMBER VARIABLES
- */
-
- private URI sessionUri;
- private Configuration sessionConfiguration;
- private int concurrentWrites = DEFAULT_CONCURRENT_WRITES;
- private boolean isAnonymousCredentials = false;
- // Set to true if we are connecting using shared access signatures.
- private boolean connectingUsingSAS = false;
- private AzureFileSystemInstrumentation instrumentation;
- private BandwidthGaugeUpdater bandwidthGaugeUpdater;
- private static final JSON PERMISSION_JSON_SERIALIZER = createPermissionJsonSerializer();
-
- private boolean suppressRetryPolicy = false;
- private boolean canCreateOrModifyContainer = false;
- private ContainerState currentKnownContainerState = ContainerState.Unknown;
- private final Object containerStateLock = new Object();
-
- private boolean tolerateOobAppends = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND;
-
- private long hadoopBlockSize = DEFAULT_HADOOP_BLOCK_SIZE;
- private int downloadBlockSizeBytes = DEFAULT_DOWNLOAD_BLOCK_SIZE;
- private int uploadBlockSizeBytes = DEFAULT_UPLOAD_BLOCK_SIZE;
- private int inputStreamVersion = DEFAULT_INPUT_STREAM_VERSION;
-
- // Bandwidth throttling exponential back-off parameters
- //
- private int minBackoff; // the minimum back-off interval (ms) between retries.
- private int maxBackoff; // the maximum back-off interval (ms) between retries.
- private int deltaBackoff; // the back-off interval (ms) between retries.
- private int maxRetries; // the maximum number of retry attempts.
-
- // Self-throttling parameters
- private boolean selfThrottlingEnabled;
- private float selfThrottlingReadFactor;
- private float selfThrottlingWriteFactor;
-
- private boolean autoThrottlingEnabled;
-
- private TestHookOperationContext testHookOperationContext = null;
-
- // Set if we're running against a storage emulator..
- private boolean isStorageEmulator = false;
-
- // Configs controlling WASB SAS key mode.
- private boolean useSecureMode = false;
- private boolean useLocalSasKeyMode = false;
-
- // User-Agent
- private String userAgentId;
-
- private String delegationToken;
-
- private boolean metadataKeyCaseSensitive;
-
- /** The error message template when container is not accessible. */
- public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
- + "account %s in the configuration, and its container %s is not "
- + "accessible using anonymous credentials. Please check if the container "
- + "exists first. If it is not publicly available, you have to provide "
- + "account credentials.";
-
- /**
- * A test hook interface that can modify the operation context we use for
- * Azure Storage operations, e.g. to inject errors.
- */
- @VisibleForTesting
- interface TestHookOperationContext {
- OperationContext modifyOperationContext(OperationContext original);
- }
-
- /**
- * Suppress the default retry policy for the Storage, useful in unit tests to
- * test negative cases without waiting forever.
- */
- @VisibleForTesting
- void suppressRetryPolicy() {
- suppressRetryPolicy = true;
- }
-
- /**
- * Add a test hook to modify the operation context we use for Azure Storage
- * operations.
- *
- * @param testHook
- * The test hook, or null to unset previous hooks.
- */
- @VisibleForTesting
- void addTestHookToOperationContext(TestHookOperationContext testHook) {
- this.testHookOperationContext = testHook;
- }
-
- /**
- * If we're asked by unit tests to not retry, set the retry policy factory in
- * the client accordingly.
- */
- private void suppressRetryPolicyInClientIfNeeded() {
- if (suppressRetryPolicy) {
- storageInteractionLayer.setRetryPolicyFactory(new RetryNoRetry());
- }
- }
-
- /**
- * Creates a JSON serializer that can serialize a PermissionStatus object into
- * the JSON string we want in the blob metadata.
- *
- * @return The JSON serializer.
- */
- private static JSON createPermissionJsonSerializer() {
- org.eclipse.jetty.util.log.Log.getProperties().setProperty("org.eclipse.jetty.util.log.announce", "false");
- JSON serializer = new JSON();
- serializer.addConvertor(PermissionStatus.class,
- new PermissionStatusJsonSerializer());
- return serializer;
- }
-
- /**
- * A converter for PermissionStatus to/from JSON as we want it in the blob
- * metadata.
- */
- private static class PermissionStatusJsonSerializer implements JSON.Convertor {
- private static final String OWNER_TAG = "owner";
- private static final String GROUP_TAG = "group";
- private static final String PERMISSIONS_TAG = "permissions";
-
- @Override
- public void toJSON(Object obj, JSON.Output out) {
- PermissionStatus permissionStatus = (PermissionStatus) obj;
- // Don't store group as null, just store it as empty string
- // (which is FileStatus behavior).
- String group = permissionStatus.getGroupName() == null ? ""
- : permissionStatus.getGroupName();
- out.add(OWNER_TAG, permissionStatus.getUserName());
- out.add(GROUP_TAG, group);
- out.add(PERMISSIONS_TAG, permissionStatus.getPermission().toString());
- }
-
- @Override
- public Object fromJSON(@SuppressWarnings("rawtypes") Map object) {
- return PermissionStatusJsonSerializer.fromJSONMap(object);
- }
-
- @SuppressWarnings("rawtypes")
- public static PermissionStatus fromJSONString(String jsonString) {
- // The JSON class can only find out about an object's class (and call me)
- // if we store the class name in the JSON string. Since I don't want to
- // do that (it's an implementation detail), I just deserialize as a
- // the default Map (JSON's default behavior) and parse that.
- return fromJSONMap((Map) PERMISSION_JSON_SERIALIZER.fromJSON(jsonString));
- }
-
- private static PermissionStatus fromJSONMap(
- @SuppressWarnings("rawtypes") Map object) {
- return new PermissionStatus((String) object.get(OWNER_TAG),
- (String) object.get(GROUP_TAG),
- // The initial - below is the Unix file type,
- // which FsPermission needs there but ignores.
- FsPermission.valueOf("-" + (String) object.get(PERMISSIONS_TAG)));
- }
- }
-
- @VisibleForTesting
- void setAzureStorageInteractionLayer(StorageInterface storageInteractionLayer) {
- this.storageInteractionLayer = storageInteractionLayer;
- }
-
- @VisibleForTesting
- public BandwidthGaugeUpdater getBandwidthGaugeUpdater() {
- return bandwidthGaugeUpdater;
- }
-
- /**
- * Check if concurrent reads and writes on the same blob are allowed.
- *
- * @return true if concurrent reads and OOB writes has been configured, false
- * otherwise.
- */
- private boolean isConcurrentOOBAppendAllowed() {
- return tolerateOobAppends;
- }
-
- /**
- * Method for the URI and configuration object necessary to create a storage
- * session with an Azure session. It parses the scheme to ensure it matches
- * the storage protocol supported by this file system.
- *
- * @param uri - URI for target storage blob.
- * @param conf - reference to configuration object.
- * @param instrumentation - the metrics source that will keep track of operations here.
- *
- * @throws IllegalArgumentException if URI or job object is null, or invalid scheme.
- */
- @Override
- public void initialize(URI uri, Configuration conf, AzureFileSystemInstrumentation instrumentation)
- throws IllegalArgumentException, AzureException, IOException {
-
- if (null == instrumentation) {
- throw new IllegalArgumentException("Null instrumentation");
- }
- this.instrumentation = instrumentation;
-
- // Check that URI exists.
- //
- if (null == uri) {
- throw new IllegalArgumentException(
- "Cannot initialize WASB file system, URI is null");
- }
-
- // Check that configuration object is non-null.
- //
- if (null == conf) {
- throw new IllegalArgumentException(
- "Cannot initialize WASB file system, conf is null");
- }
-
- if (!conf.getBoolean(
- NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
- //If not skip azure metrics, create bandwidthGaugeUpdater
- this.bandwidthGaugeUpdater = new BandwidthGaugeUpdater(instrumentation);
- }
-
- // Incoming parameters validated. Capture the URI and the job configuration
- // object.
- //
- sessionUri = uri;
- sessionConfiguration = conf;
-
- useSecureMode = conf.getBoolean(KEY_USE_SECURE_MODE,
- DEFAULT_USE_SECURE_MODE);
- useLocalSasKeyMode = conf.getBoolean(KEY_USE_LOCAL_SAS_KEY_MODE,
- DEFAULT_USE_LOCAL_SAS_KEY_MODE);
-
- if (null == this.storageInteractionLayer) {
- if (!useSecureMode) {
- this.storageInteractionLayer = new StorageInterfaceImpl();
- } else {
- this.storageInteractionLayer = new SecureStorageInterfaceImpl(
- useLocalSasKeyMode, conf);
- }
- }
-
- // Configure Azure storage session.
- configureAzureStorageSession();
-
- // Start an Azure storage session.
- //
- createAzureStorageSession();
-
- // Extract the directories that should contain page blobs
- pageBlobDirs = getDirectorySet(KEY_PAGE_BLOB_DIRECTORIES);
- LOG.debug("Page blob directories: {}", setToString(pageBlobDirs));
-
- // User-agent
- userAgentId = conf.get(USER_AGENT_ID_KEY, USER_AGENT_ID_DEFAULT);
-
- // Extract the directories that should contain block blobs with compaction
- blockBlobWithCompationDirs = getDirectorySet(
- KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES);
- LOG.debug("Block blobs with compaction directories: {}",
- setToString(blockBlobWithCompationDirs));
-
- // Extract directories that should have atomic rename applied.
- atomicRenameDirs = getDirectorySet(KEY_ATOMIC_RENAME_DIRECTORIES);
- String hbaseRoot;
- try {
-
- // Add to this the hbase root directory, or /hbase is that is not set.
- hbaseRoot = verifyAndConvertToStandardFormat(
- sessionConfiguration.get("hbase.rootdir", "hbase"));
- if (hbaseRoot != null) {
- atomicRenameDirs.add(hbaseRoot);
- }
- } catch (URISyntaxException e) {
- LOG.warn("Unable to initialize HBase root as an atomic rename directory.");
- }
- LOG.debug("Atomic rename directories: {} ", setToString(atomicRenameDirs));
- metadataKeyCaseSensitive = conf
- .getBoolean(KEY_BLOB_METADATA_KEY_CASE_SENSITIVE, true);
- if (!metadataKeyCaseSensitive) {
- LOG.info("{} configured as false. Blob metadata will be treated case insensitive.",
- KEY_BLOB_METADATA_KEY_CASE_SENSITIVE);
- }
- }
-
- /**
- * Helper to format a string for log output from Set<String>
- */
- private String setToString(Set<String> set) {
- StringBuilder sb = new StringBuilder();
- int i = 1;
- for (String s : set) {
- sb.append("/" + s);
- if (i != set.size()) {
- sb.append(", ");
- }
- i++;
- }
- return sb.toString();
- }
-
- /**
- * Method to extract the account name from an Azure URI.
- *
- * @param uri
- * -- WASB blob URI
- * @returns accountName -- the account name for the URI.
- * @throws URISyntaxException
- * if the URI does not have an authority it is badly formed.
- */
- private String getAccountFromAuthority(URI uri) throws URISyntaxException {
-
- // Check to make sure that the authority is valid for the URI.
- //
- String authority = uri.getRawAuthority();
- if (null == authority) {
- // Badly formed or illegal URI.
- //
- throw new URISyntaxException(uri.toString(),
- "Expected URI with a valid authority");
- }
-
- // Check if authority container the delimiter separating the account name from the
- // the container.
- //
- if (!authority.contains(WASB_AUTHORITY_DELIMITER)) {
- return authority;
- }
-
- // Split off the container name and the authority.
- //
- String[] authorityParts = authority.split(WASB_AUTHORITY_DELIMITER, 2);
-
- // Because the string contains an '@' delimiter, a container must be
- // specified.
- //
- if (authorityParts.length < 2 || "".equals(authorityParts[0])) {
- // Badly formed WASB authority since there is no container.
- //
- final String errMsg = String
- .format(
- "URI '%s' has a malformed WASB authority, expected container name. "
- + "Authority takes the form wasb://[<container name>@]<account name>",
- uri.toString());
- throw new IllegalArgumentException(errMsg);
- }
-
- // Return with the account name. It is possible that this name is NULL.
- //
- return authorityParts[1];
- }
-
- /**
- * Method to extract the container name from an Azure URI.
- *
- * @param uri
- * -- WASB blob URI
- * @returns containerName -- the container name for the URI. May be null.
- * @throws URISyntaxException
- * if the uri does not have an authority it is badly formed.
- */
- private String getContainerFromAuthority(URI uri) throws URISyntaxException {
-
- // Check to make sure that the authority is valid for the URI.
- //
- String authority = uri.getRawAuthority();
- if (null == authority) {
- // Badly formed or illegal URI.
- //
- throw new URISyntaxException(uri.toString(),
- "Expected URI with a valid authority");
- }
-
- // The URI has a valid authority. Extract the container name. It is the
- // second component of the WASB URI authority.
- if (!authority.contains(WASB_AUTHORITY_DELIMITER)) {
- // The authority does not have a container name. Use the default container by
- // setting the container name to the default Azure root container.
- //
- return AZURE_ROOT_CONTAINER;
- }
-
- // Split off the container name and the authority.
- String[] authorityParts = authority.split(WASB_AUTHORITY_DELIMITER, 2);
-
- // Because the string contains an '@' delimiter, a container must be
- // specified.
- if (authorityParts.length < 2 || "".equals(authorityParts[0])) {
- // Badly formed WASB authority since there is no container.
- final String errMsg = String
- .format(
- "URI '%s' has a malformed WASB authority, expected container name."
- + "Authority takes the form wasb://[<container name>@]<account name>",
- uri.toString());
- throw new IllegalArgumentException(errMsg);
- }
-
- // Set the container name from the first entry for the split parts of the
- // authority.
- return authorityParts[0];
- }
-
- /**
- * Get the appropriate return the appropriate scheme for communicating with
- * Azure depending on whether wasb or wasbs is specified in the target URI.
- *
- * return scheme - HTTPS or HTTP as appropriate.
- */
- private String getHTTPScheme() {
- String sessionScheme = sessionUri.getScheme();
- // Check if we're on a secure URI scheme: wasbs or the legacy asvs scheme.
- if (sessionScheme != null
- && (sessionScheme.equalsIgnoreCase("asvs")
- || sessionScheme.equalsIgnoreCase("wasbs"))) {
- return HTTPS_SCHEME;
- } else {
- // At this point the scheme should be either null or asv or wasb.
- // Intentionally I'm not going to validate it though since I don't feel
- // it's this method's job to ensure a valid URI scheme for this file
- // system.
- return HTTP_SCHEME;
- }
- }
-
- /**
- * Set the configuration parameters for this client storage session with
- * Azure.
- *
- * @throws AzureException
- */
- private void configureAzureStorageSession() throws AzureException {
-
- // Assertion: Target session URI already should have been captured.
- if (sessionUri == null) {
- throw new AssertionError(
- "Expected a non-null session URI when configuring storage session");
- }
-
- // Assertion: A client session already should have been established with
- // Azure.
- if (storageInteractionLayer == null) {
- throw new AssertionError(String.format(
- "Cannot configure storage session for URI '%s' "
- + "if storage session has not been established.",
- sessionUri.toString()));
- }
-
- // Determine whether or not reads are allowed concurrent with OOB writes.
- tolerateOobAppends = sessionConfiguration.getBoolean(
- KEY_READ_TOLERATE_CONCURRENT_APPEND,
- DEFAULT_READ_TOLERATE_CONCURRENT_APPEND);
-
- // Retrieve configuration for the minimum stream read and write block size.
- //
- this.downloadBlockSizeBytes = sessionConfiguration.getInt(
- KEY_STREAM_MIN_READ_SIZE, DEFAULT_DOWNLOAD_BLOCK_SIZE);
- this.uploadBlockSizeBytes = sessionConfiguration.getInt(
- KEY_WRITE_BLOCK_SIZE, DEFAULT_UPLOAD_BLOCK_SIZE);
- this.hadoopBlockSize = sessionConfiguration.getLong(
- HADOOP_BLOCK_SIZE_PROPERTY_NAME, DEFAULT_HADOOP_BLOCK_SIZE);
-
- this.inputStreamVersion = sessionConfiguration.getInt(
- KEY_INPUT_STREAM_VERSION, DEFAULT_INPUT_STREAM_VERSION);
-
- // The job may want to specify a timeout to use when engaging the
- // storage service. The default is currently 90 seconds. It may
- // be necessary to increase this value for long latencies in larger
- // jobs. If the timeout specified is greater than zero seconds use
- // it, otherwise use the default service client timeout.
- int storageConnectionTimeout = sessionConfiguration.getInt(
- KEY_STORAGE_CONNECTION_TIMEOUT, 0);
-
- if (0 < storageConnectionTimeout) {
- storageInteractionLayer.setTimeoutInMs(storageConnectionTimeout * 1000);
- }
-
- // Set the concurrency values equal to the that specified in the
- // configuration file. If it does not exist, set it to the default
- // value calculated as double the number of CPU cores on the client
- // machine. The concurrency value is minimum of double the cores and
- // the read/write property.
- int cpuCores = 2 * Runtime.getRuntime().availableProcessors();
-
- concurrentWrites = sessionConfiguration.getInt(
- KEY_CONCURRENT_CONNECTION_VALUE_OUT,
- Math.min(cpuCores, DEFAULT_CONCURRENT_WRITES));
-
- // Set up the exponential retry policy.
- //
- minBackoff = sessionConfiguration.getInt(
- KEY_MIN_BACKOFF_INTERVAL, DEFAULT_MIN_BACKOFF_INTERVAL);
-
- maxBackoff = sessionConfiguration.getInt(
- KEY_MAX_BACKOFF_INTERVAL, DEFAULT_MAX_BACKOFF_INTERVAL);
-
- deltaBackoff = sessionConfiguration.getInt(
- KEY_BACKOFF_INTERVAL, DEFAULT_BACKOFF_INTERVAL);
-
- maxRetries = sessionConfiguration.getInt(
- KEY_MAX_IO_RETRIES, DEFAULT_MAX_RETRY_ATTEMPTS);
-
- storageInteractionLayer.setRetryPolicyFactory(
- new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries));
-
-
- // read the self-throttling config.
- selfThrottlingEnabled = sessionConfiguration.getBoolean(
- KEY_SELF_THROTTLE_ENABLE, DEFAULT_SELF_THROTTLE_ENABLE);
-
- selfThrottlingReadFactor = sessionConfiguration.getFloat(
- KEY_SELF_THROTTLE_READ_FACTOR, DEFAULT_SELF_THROTTLE_READ_FACTOR);
-
- selfThrottlingWriteFactor = sessionConfiguration.getFloat(
- KEY_SELF_THROTTLE_WRITE_FACTOR, DEFAULT_SELF_THROTTLE_WRITE_FACTOR);
-
- if (!selfThrottlingEnabled) {
- autoThrottlingEnabled = sessionConfiguration.getBoolean(
- KEY_AUTO_THROTTLE_ENABLE,
- DEFAULT_AUTO_THROTTLE_ENABLE);
- if (autoThrottlingEnabled) {
- ClientThrottlingIntercept.initializeSingleton();
- }
- } else {
- // cannot enable both self-throttling and client-throttling
- autoThrottlingEnabled = false;
- }
-
- OperationContext.setLoggingEnabledByDefault(sessionConfiguration.
- getBoolean(KEY_ENABLE_STORAGE_CLIENT_LOGGING, false));
-
- LOG.debug(
- "AzureNativeFileSystemStore init. Settings={},{},{},{{},{},{},{}},{{},{},{}}",
- concurrentWrites, tolerateOobAppends,
- ((storageConnectionTimeout > 0) ? storageConnectionTimeout
- : STORAGE_CONNECTION_TIMEOUT_DEFAULT), minBackoff,
- deltaBackoff, maxBackoff, maxRetries, selfThrottlingEnabled,
- selfThrottlingReadFactor, selfThrottlingWriteFactor);
- }
-
- /**
- * Connect to Azure storage using anonymous credentials.
- *
- * @param uri
- * - URI to target blob (R/O access to public blob)
- *
- * @throws StorageException
- * raised on errors communicating with Azure storage.
- * @throws IOException
- * raised on errors performing I/O or setting up the session.
- * @throws URISyntaxException
- * raised on creating mal-formed URI's.
- */
- private void connectUsingAnonymousCredentials(final URI uri)
- throws StorageException, IOException, URISyntaxException {
- // Use an HTTP scheme since the URI specifies a publicly accessible
- // container. Explicitly create a storage URI corresponding to the URI
- // parameter for use in creating the service client.
- String accountName = getAccountFromAuthority(uri);
- URI storageUri = new URI(getHTTPScheme() + ":" + PATH_DELIMITER
- + PATH_DELIMITER + accountName);
-
- // Create the service client with anonymous credentials.
- String containerName = getContainerFromAuthority(uri);
- storageInteractionLayer.createBlobClient(storageUri);
- suppressRetryPolicyInClientIfNeeded();
-
- // Capture the container reference.
- container = storageInteractionLayer.getContainerReference(containerName);
- rootDirectory = container.getDirectoryReference("");
-
- // Check for container existence, and our ability to access it.
- boolean canAccess;
- try {
- canAccess = container.exists(getInstrumentedContext());
- } catch (StorageException ex) {
- LOG.error("Service returned StorageException when checking existence "
- + "of container {} in account {}", containerName, accountName, ex);
- canAccess = false;
- }
- if (!canAccess) {
- throw new AzureException(String.format(NO_ACCESS_TO_CONTAINER_MSG,
- accountName, containerName));
- }
-
- // Accessing the storage server unauthenticated using
- // anonymous credentials.
- isAnonymousCredentials = true;
- }
-
- private void connectUsingCredentials(String accountName,
- StorageCredentials credentials, String containerName)
- throws URISyntaxException, StorageException, AzureException {
-
- URI blobEndPoint;
- if (isStorageEmulatorAccount(accountName)) {
- isStorageEmulator = true;
- CloudStorageAccount account =
- CloudStorageAccount.getDevelopmentStorageAccount();
- storageInteractionLayer.createBlobClient(account);
- } else {
- blobEndPoint = new URI(getHTTPScheme() + "://" + accountName);
- storageInteractionLayer.createBlobClient(blobEndPoint, credentials);
- }
- suppressRetryPolicyInClientIfNeeded();
-
- // Capture the container reference for debugging purposes.
- container = storageInteractionLayer.getContainerReference(containerName);
- rootDirectory = container.getDirectoryReference("");
-
- // Can only create container if using account key credentials
- canCreateOrModifyContainer = credentials instanceof StorageCredentialsAccountAndKey;
- }
-
- /**
- * Method to set up the Storage Interaction layer in Secure mode.
- * @param accountName - Storage account provided in the initializer
- * @param containerName - Container name provided in the initializer
- * @param sessionUri - URI provided in the initializer
- */
- private void connectToAzureStorageInSecureMode(String accountName,
- String containerName, URI sessionUri)
- throws AzureException, StorageException, URISyntaxException {
-
- LOG.debug("Connecting to Azure storage in Secure Mode");
- // Assertion: storageInteractionLayer instance has to be a SecureStorageInterfaceImpl
- if (!(this.storageInteractionLayer instanceof SecureStorageInterfaceImpl)) {
- throw new AssertionError("connectToAzureStorageInSecureMode() should be called only"
- + " for SecureStorageInterfaceImpl instances");
- }
-
- ((SecureStorageInterfaceImpl) this.storageInteractionLayer).
- setStorageAccountName(accountName);
- connectingUsingSAS = true;
- container = storageInteractionLayer.getContainerReference(containerName);
- rootDirectory = container.getDirectoryReference("");
-
- canCreateOrModifyContainer = true;
- }
-
- /**
- * Connect to Azure storage using account key credentials.
- */
- private void connectUsingConnectionStringCredentials(
- final String accountName, final String containerName,
- final String accountKey) throws InvalidKeyException, StorageException,
- IOException, URISyntaxException {
- // If the account name is "acc.blob.core.windows.net", then the
- // rawAccountName is just "acc"
- String rawAccountName = accountName.split("\\.")[0];
- StorageCredentials credentials = new StorageCredentialsAccountAndKey(
- rawAccountName, accountKey);
- connectUsingCredentials(accountName, credentials, containerName);
- }
-
- /**
- * Connect to Azure storage using shared access signature credentials.
- */
- private void connectUsingSASCredentials(final String accountName,
- final String containerName, final String sas) throws InvalidKeyException,
- StorageException, IOException, URISyntaxException {
- StorageCredentials credentials = new StorageCredentialsSharedAccessSignature(
- sas);
- connectingUsingSAS = true;
- connectUsingCredentials(accountName, credentials, containerName);
- }
-
- private boolean isStorageEmulatorAccount(final String accountName) {
- return accountName.equalsIgnoreCase(sessionConfiguration.get(
- STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME,
- DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME));
- }
-
- @VisibleForTesting
- public static String getAccountKeyFromConfiguration(String accountName,
- Configuration conf) throws KeyProviderException {
- String key = null;
- String keyProviderClass = conf.get(KEY_ACCOUNT_KEYPROVIDER_PREFIX
- + accountName);
- KeyProvider keyProvider = null;
-
- if (keyProviderClass == null) {
- // No key provider was provided so use the provided key as is.
- keyProvider = new SimpleKeyProvider();
- } else {
- // create an instance of the key provider class and verify it
- // implements KeyProvider
- Object keyProviderObject = null;
- try {
- Class<?> clazz = conf.getClassByName(keyProviderClass);
- keyProviderObject = clazz.newInstance();
- } catch (Exception e) {
- throw new KeyProviderException("Unable to load key provider class.", e);
- }
- if (!(keyProviderObject instanceof KeyProvider)) {
- throw new KeyProviderException(keyProviderClass
- + " specified in config is not a valid KeyProvider class.");
- }
- keyProvider = (KeyProvider) keyProviderObject;
- }
- key = keyProvider.getStorageAccountKey(accountName, conf);
-
- return key;
- }
-
- /**
- * Establish a session with Azure blob storage based on the target URI. The
- * method determines whether or not the URI target contains an explicit
- * account or an implicit default cluster-wide account.
- *
- * @throws AzureException
- * @throws IOException
- */
- private void createAzureStorageSession()
- throws AzureException, IOException {
-
- // Make sure this object was properly initialized with references to
- // the sessionUri and sessionConfiguration.
- if (null == sessionUri || null == sessionConfiguration) {
- throw new AzureException("Filesystem object not initialized properly."
- + "Unable to start session with Azure Storage server.");
- }
-
- // File system object initialized, attempt to establish a session
- // with the Azure storage service for the target URI string.
- try {
- // Inspect the URI authority to determine the account and use the account
- // to start an Azure blob client session using an account key for the
- // the account or anonymously.
- // For all URI's do the following checks in order:
- // 1. Validate that <account> can be used with the current Hadoop
- // cluster by checking it exists in the list of configured accounts
- // for the cluster.
- // 2. Look up the AccountKey in the list of configured accounts for the
- // cluster.
- // 3. If there is no AccountKey, assume anonymous public blob access
- // when accessing the blob.
- //
- // If the URI does not specify a container use the default root container
- // under the account name.
-
- // Assertion: Container name on the session Uri should be non-null.
- if (getContainerFromAuthority(sessionUri) == null) {
- throw new AssertionError(String.format(
- "Non-null container expected from session URI: %s.",
- sessionUri.toString()));
- }
-
- // Get the account name.
- String accountName = getAccountFromAuthority(sessionUri);
- if (null == accountName) {
- // Account name is not specified as part of the URI. Throw indicating
- // an invalid account name.
- final String errMsg = String.format(
- "Cannot load WASB file system account name not"
- + " specified in URI: %s.", sessionUri.toString());
- throw new AzureException(errMsg);
- }
-
- instrumentation.setAccountName(accountName);
- String containerName = getContainerFromAuthority(sessionUri);
- instrumentation.setContainerName(containerName);
-
- // Check whether this is a storage emulator account.
- if (isStorageEmulatorAccount(accountName)) {
- // It is an emulator account, connect to it with no credentials.
- connectUsingCredentials(accountName, null, containerName);
- return;
- }
-
- // If the securemode flag is set, WASB uses SecureStorageInterfaceImpl instance
- // to communicate with Azure storage. In SecureStorageInterfaceImpl SAS keys
- // are used to communicate with Azure storage, so connectToAzureStorageInSecureMode
- // instantiates the default container using a SAS Key.
- if (useSecureMode) {
- connectToAzureStorageInSecureMode(accountName, containerName, sessionUri);
- return;
- }
-
- // Check whether we have a shared access signature for that container.
- String propertyValue = sessionConfiguration.get(KEY_ACCOUNT_SAS_PREFIX
- + containerName + "." + accountName);
- if (propertyValue != null) {
- // SAS was found. Connect using that.
- connectUsingSASCredentials(accountName, containerName, propertyValue);
- return;
- }
-
- // Check whether the account is configured with an account key.
- propertyValue = getAccountKeyFromConfiguration(accountName,
- sessionConfiguration);
- if (StringUtils.isNotEmpty(propertyValue)) {
- // Account key was found.
- // Create the Azure storage session using the account key and container.
- connectUsingConnectionStringCredentials(
- getAccountFromAuthority(sessionUri),
- getContainerFromAuthority(sessionUri), propertyValue);
- } else {
- LOG.debug("The account access key is not configured for {}. "
- + "Now try anonymous access.", sessionUri);
- connectUsingAnonymousCredentials(sessionUri);
- }
- } catch (Exception e) {
- // Caught exception while attempting to initialize the Azure File
- // System store, re-throw the exception.
- throw new AzureException(e);
- }
- }
-
- private enum ContainerState {
- /**
- * We haven't checked the container state yet.
- */
- Unknown,
- /**
- * We checked and the container doesn't exist.
- */
- DoesntExist,
- /**
- * The container exists and doesn't have an WASB version stamp on it.
- */
- ExistsNoVersion,
- /**
- * The container exists and has an unsupported WASB version stamped on it.
- */
- ExistsAtWrongVersion,
- /**
- * The container exists and has the proper WASB version stamped on it.
- */
- ExistsAtRightVersion
- }
-
- private enum ContainerAccessType {
- /**
- * We're accessing the container for a pure read operation, e.g. read a
- * file.
- */
- PureRead,
- /**
- * We're accessing the container purely to write something, e.g. write a
- * file.
- */
- PureWrite,
- /**
- * We're accessing the container to read something then write, e.g. rename a
- * file.
- */
- ReadThenWrite
- }
-
- /**
- * Trims a suffix/prefix from the given string. For example if
- * s is given as "/xy" and toTrim is "/", this method returns "xy"
- */
- private static String trim(String s, String toTrim) {
- return StringUtils.removeEnd(StringUtils.removeStart(s, toTrim),
- toTrim);
- }
-
- /**
- * Checks if the given rawDir belongs to this account/container, and
- * if so returns the canonicalized path for it. Otherwise return null.
- */
- private String verifyAndConvertToStandardFormat(String rawDir) throws URISyntaxException {
- URI asUri = new URI(rawDir);
- if (asUri.getAuthority() == null
- || asUri.getAuthority().toLowerCase(Locale.ENGLISH).equalsIgnoreCase(
- sessionUri.getAuthority().toLowerCase(Locale.ENGLISH))) {
- // Applies to me.
- return trim(asUri.getPath(), "/");
- } else {
- // Doen't apply to me.
- return null;
- }
- }
-
- /**
- * Take a comma-separated list of directories from a configuration variable
- * and transform it to a set of directories.
- */
- private Set<String> getDirectorySet(final String configVar)
- throws AzureException {
- String[] rawDirs = sessionConfiguration.getStrings(configVar, new String[0]);
- Set<String> directorySet = new HashSet<String>();
- for (String currentDir : rawDirs) {
- String myDir;
- try {
- myDir = verifyAndConvertToStandardFormat(currentDir.trim());
- } catch (URISyntaxException ex) {
- throw new AzureException(String.format(
- "The directory %s specified in the configuration entry %s is not"
- + " a valid URI.",
- currentDir, configVar));
- }
- if (myDir != null) {
- directorySet.add(myDir);
- }
- }
- return directorySet;
- }
-
- /**
- * Checks if the given key in Azure Storage should be stored as a page
- * blob instead of block blob.
- */
- public boolean isPageBlobKey(String key) {
- return isKeyForDirectorySet(key, pageBlobDirs);
- }
-
- /**
- * Checks if the given key in Azure Storage should be stored as a block blobs
- * with compaction enabled instead of normal block blob.
- *
- * @param key blob name
- * @return true, if the file is in directory with block compaction enabled.
- */
- public boolean isBlockBlobWithCompactionKey(String key) {
- return isKeyForDirectorySet(key, blockBlobWithCompationDirs);
- }
-
- /**
- * Checks if the given key in Azure storage should have synchronized
- * atomic folder rename createNonRecursive implemented.
- */
- @Override
- public boolean isAtomicRenameKey(String key) {
- return isKeyForDirectorySet(key, atomicRenameDirs);
- }
-
- public boolean isKeyForDirectorySet(String key, Set<String> dirSet) {
- String defaultFS = FileSystem.getDefaultUri(sessionConfiguration).toString();
- for (String dir : dirSet) {
- if (dir.isEmpty()) {
- // dir is root
- return true;
- }
-
- if (matchAsteriskPattern(key, dir)) {
- return true;
- }
-
- // Allow for blob directories with paths relative to the default file
- // system.
- //
- try {
- URI uriPageBlobDir = new URI(dir);
- if (null == uriPageBlobDir.getAuthority()) {
- // Concatenate the default file system prefix with the relative
- // page blob directory path.
- //
- String dirWithPrefix = trim(defaultFS, "/") + "/" + dir;
- if (matchAsteriskPattern(key, dirWithPrefix)) {
- return true;
- }
- }
- } catch (URISyntaxException e) {
- LOG.info("URI syntax error creating URI for {}", dir);
- }
- }
- return false;
- }
-
- private boolean matchAsteriskPattern(String pathName, String pattern) {
- if (pathName == null || pathName.length() == 0) {
- return false;
- }
-
- int pathIndex = 0;
- int patternIndex = 0;
-
- while (pathIndex < pathName.length() && patternIndex < pattern.length()) {
- char charToMatch = pattern.charAt(patternIndex);
-
- // normal char:
- if (charToMatch != ASTERISK_SYMBOL) {
- if (charToMatch != pathName.charAt(pathIndex)) {
- return false;
- }
- pathIndex++;
- patternIndex++;
- continue;
- }
-
- // ASTERISK_SYMBOL
- // 1. * is used in path name: *a/b,a*/b, a/*b, a/b*
- if (patternIndex > 0 && pattern.charAt(patternIndex - 1) != Path.SEPARATOR_CHAR
- || patternIndex + 1 < pattern.length() && pattern.charAt(patternIndex + 1) != Path.SEPARATOR_CHAR) {
- if (ASTERISK_SYMBOL != pathName.charAt(pathIndex)) {
- return false;
- }
-
- pathIndex++;
- patternIndex++;
- continue;
- }
-
- // 2. * is used as wildcard: */a, a/*/b, a/*
- patternIndex++;
- // find next path separator
- while (pathIndex < pathName.length() && pathName.charAt(pathIndex) != Path.SEPARATOR_CHAR) {
- pathIndex++;
- }
- }
-
- // Ensure it is not a file/dir which shares same prefix as pattern
- // Eg: pattern: /A/B, pathName: /A/BBB should not match
- return patternIndex == pattern.length()
- && (pathIndex == pathName.length() || pathName.charAt(pathIndex) == Path.SEPARATOR_CHAR);
- }
-
- /**
- * Returns the file block size. This is a fake value used for integration
- * of the Azure store with Hadoop.
- */
- @Override
- public long getHadoopBlockSize() {
- return hadoopBlockSize;
- }
-
- /**
- * This should be called from any method that does any modifications to the
- * underlying container: it makes sure to put the WASB current version in the
- * container's metadata if it's not already there.
- */
- private ContainerState checkContainer(ContainerAccessType accessType)
- throws StorageException, AzureException {
- synchronized (containerStateLock) {
- if (isOkContainerState(accessType)) {
- return currentKnownContainerState;
- }
- if (currentKnownContainerState == ContainerState.ExistsAtWrongVersion) {
- String containerVersion = retrieveVersionAttribute(container);
- throw wrongVersionException(containerVersion);
- }
- // This means I didn't check it before or it didn't exist or
- // we need to stamp the version. Since things may have changed by
- // other machines since then, do the check again and don't depend
- // on past information.
-
- // Sanity check: we don't expect this at this point.
- if (currentKnownContainerState == ContainerState.ExistsAtRightVersion) {
- throw new AssertionError("Unexpected state: "
- + currentKnownContainerState);
- }
-
- // Download the attributes - doubles as an existence check with just
- // one service call
- try {
- container.downloadAttributes(getInstrumentedContext());
- currentKnownContainerState = ContainerState.Unknown;
- } catch (StorageException ex) {
- if (StorageErrorCodeStrings.CONTAINER_NOT_FOUND.toString()
- .equals(ex.getErrorCode())) {
- currentKnownContainerState = ContainerState.DoesntExist;
- } else {
- throw ex;
- }
- }
-
- if (currentKnownContainerState == ContainerState.DoesntExist) {
- // If the container doesn't exist and we intend to write to it,
- // create it now.
- if (needToCreateContainer(accessType)) {
- storeVersionAttribute(container);
- container.create(getInstrumentedContext());
- currentKnownContainerState = ContainerState.ExistsAtRightVersion;
- }
- } else {
- // The container exists, check the version.
- String containerVersion = retrieveVersionAttribute(container);
- if (containerVersion != null) {
- if (containerVersion.equals(FIRST_WASB_VERSION)) {
- // It's the version from when WASB was called ASV, just
- // fix the version attribute if needed and proceed.
- // We should be good otherwise.
- if (needToStampVersion(accessType)) {
- storeVersionAttribute(container);
- container.uploadMetadata(getInstrumentedContext());
- }
- } else if (!containerVersion.equals(CURRENT_WASB_VERSION)) {
- // Don't know this version - throw.
- currentKnownContainerState = ContainerState.ExistsAtWrongVersion;
- throw wrongVersionException(containerVersion);
- } else {
- // It's our correct version.
- currentKnownContainerState = ContainerState.ExistsAtRightVersion;
- }
- } else {
- // No version info exists.
- currentKnownContainerState = ContainerState.ExistsNoVersion;
- if (needToStampVersion(accessType)) {
- // Need to stamp the version
- storeVersionAttribute(container);
- container.uploadMetadata(getInstrumentedContext());
- currentKnownContainerState = ContainerState.ExistsAtRightVersion;
- }
- }
- }
- return currentKnownContainerState;
- }
- }
-
- private AzureException wrongVersionException(String containerVersion) {
- return new AzureException("The container " + container.getName()
- + " is at an unsupported version: " + containerVersion
- + ". Current supported version: " + FIRST_WASB_VERSION);
- }
-
- private boolean needToStampVersion(ContainerAccessType accessType) {
- // We need to stamp the version on the container any time we write to
- // it and we have the correct credentials to be able to write container
- // metadata.
- return accessType != ContainerAccessType.PureRead
- && canCreateOrModifyContainer;
- }
-
- private static boolean needToCreateContainer(ContainerAccessType accessType) {
- // We need to pro-actively create the container (if it doesn't exist) if
- // we're doing a pure write. No need to create it for pure read or read-
- // then-write access.
- return accessType == ContainerAccessType.PureWrite;
- }
-
- // Determines whether we have to pull the container information again
- // or we can work based off what we already have.
- private boolean isOkContainerState(ContainerAccessType accessType) {
- switch (currentKnownContainerState) {
- case Unknown:
- // When using SAS, we can't discover container attributes
- // so just live with Unknown state and fail later if it
- // doesn't exist.
- return connectingUsingSAS;
- case DoesntExist:
- return false; // the container could have been created
- case ExistsAtRightVersion:
- return true; // fine to optimize
- case ExistsAtWrongVersion:
- return false;
- case ExistsNoVersion:
- // If there's no version, it's OK if we don't need to stamp the version
- // or we can't anyway even if we wanted to.
- return !needToStampVersion(accessType);
- default:
- throw new AssertionError("Unknown access type: " + accessType);
- }
- }
-
- private boolean getUseTransactionalContentMD5() {
- return sessionConfiguration.getBoolean(KEY_CHECK_BLOCK_MD5, true);
- }
-
- private BlobRequestOptions getUploadOptions() {
- BlobRequestOptions options = new BlobRequestOptions();
- options.setStoreBlobContentMD5(sessionConfiguration.getBoolean(
- KEY_STORE_BLOB_MD5, false));
- options.setUseTransactionalContentMD5(getUseTransactionalContentMD5());
- options.setConcurrentRequestCount(concurrentWrites);
-
- options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff,
- deltaBackoff, maxBackoff, maxRetries));
-
- return options;
- }
-
- private BlobRequestOptions getDownloadOptions() {
- BlobRequestOptions options = new BlobRequestOptions();
- options.setRetryPolicyFactory(
- new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries));
- options.setUseTransactionalContentMD5(getUseTransactionalContentMD5());
- return options;
- }
-
- @Override
- public DataOutputStream storefile(String keyEncoded,
- PermissionStatus permissionStatus,
- String key)
- throws AzureException {
- try {
-
- // Check if a session exists, if not create a session with the
- // Azure storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AzureException(errMsg);
- }
-
- // Check if there is an authenticated account associated with the
- // file this instance of the WASB file system. If not the file system
- // has not been authenticated and all access is anonymous.
- if (!isAuthenticatedAccess()) {
- // Preemptively raise an exception indicating no uploads are
- // allowed to anonymous accounts.
- throw new AzureException(new IOException(
- "Uploads to public accounts using anonymous "
- + "access is prohibited."));
- }
-
- checkContainer(ContainerAccessType.PureWrite);
-
- /**
- * Note: Windows Azure Blob Storage does not allow the creation of arbitrary directory
- * paths under the default $root directory. This is by design to eliminate
- * ambiguity in specifying a implicit blob address. A blob in the $root conatiner
- * cannot include a / in its name and must be careful not to include a trailing
- * '/' when referencing blobs in the $root container.
- * A '/; in the $root container permits ambiguous blob names as in the following
- * example involving two containers $root and mycontainer:
- * http://myaccount.blob.core.windows.net/$root
- * http://myaccount.blob.core.windows.net/mycontainer
- * If the URL "mycontainer/somefile.txt were allowed in $root then the URL:
- * http://myaccount.blob.core.windows.net/mycontainer/myblob.txt
- * could mean either:
- * (1) container=mycontainer; blob=myblob.txt
- * (2) container=$root; blob=mycontainer/myblob.txt
- *
- * To avoid this type of ambiguity the Azure blob storage prevents
- * arbitrary path under $root. For a simple and more consistent user
- * experience it was decided to eliminate the opportunity for creating
- * such paths by making the $root container read-only under WASB.
- */
-
- // Check that no attempt is made to write to blobs on default
- // $root containers.
- if (AZURE_ROOT_CONTAINER.equals(getContainerFromAuthority(sessionUri))) {
- // Azure containers are restricted to non-root containers.
- final String errMsg = String.format(
- "Writes to '%s' container for URI '%s' are prohibited, "
- + "only updates on non-root containers permitted.",
- AZURE_ROOT_CONTAINER, sessionUri.toString());
- throw new AzureException(errMsg);
- }
-
- // Get the blob reference from the store's container and
- // return it.
- CloudBlobWrapper blob = getBlobReference(keyEncoded);
- storePermissionStatus(blob, permissionStatus);
-
- // Create the output stream for the Azure blob.
- //
- OutputStream outputStream;
-
- if (isBlockBlobWithCompactionKey(key)) {
- BlockBlobAppendStream blockBlobOutputStream = new BlockBlobAppendStream(
- (CloudBlockBlobWrapper) blob,
- keyEncoded,
- this.uploadBlockSizeBytes,
- true,
- getInstrumentedContext());
-
- outputStream = blockBlobOutputStream;
- } else {
- outputStream = openOutputStream(blob);
- }
-
- DataOutputStream dataOutStream = new SyncableDataOutputStream(outputStream);
- return dataOutStream;
- } catch (Exception e) {
- // Caught exception while attempting to open the blob output stream.
- // Re-throw as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- /**
- * Opens a new output stream to the given blob (page or block blob)
- * to populate it from scratch with data.
- */
- private OutputStream openOutputStream(final CloudBlobWrapper blob)
- throws StorageException {
- if (blob instanceof CloudPageBlobWrapper){
- return new PageBlobOutputStream(
- (CloudPageBlobWrapper) blob, getInstrumentedContext(), sessionConfiguration);
- } else {
-
- // Handle both ClouldBlockBlobWrapperImpl and (only for the test code path)
- // MockCloudBlockBlobWrapper.
- return ((CloudBlockBlobWrapper) blob).openOutputStream(getUploadOptions(),
- getInstrumentedContext());
- }
- }
-
- /**
- * Opens a new input stream for the given blob (page or block blob)
- * to read its data.
- */
- private InputStream openInputStream(CloudBlobWrapper blob,
- Optional<Configuration> options) throws StorageException, IOException {
- if (blob instanceof CloudBlockBlobWrapper) {
- LOG.debug("Using stream seek algorithm {}", inputStreamVersion);
- switch(inputStreamVersion) {
- case 1:
- return blob.openInputStream(getDownloadOptions(),
- getInstrumentedContext(isConcurrentOOBAppendAllowed()));
- case 2:
- boolean bufferedPreadDisabled = options.map(c -> c
- .getBoolean(FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE, false))
- .orElse(false);
- return new BlockBlobInputStream((CloudBlockBlobWrapper) blob,
- getDownloadOptions(),
- getInstrumentedContext(isConcurrentOOBAppendAllowed()),
- bufferedPreadDisabled);
- default:
- throw new IOException("Unknown seek algorithm: " + inputStreamVersion);
- }
- } else {
- return new PageBlobInputStream(
- (CloudPageBlobWrapper) blob, getInstrumentedContext(
- isConcurrentOOBAppendAllowed()));
- }
- }
-
- /**
- * Default permission to use when no permission metadata is found.
- *
- * @return The default permission to use.
- */
- private static PermissionStatus defaultPermissionNoBlobMetadata() {
- return new PermissionStatus("", "", FsPermission.getDefault());
- }
-
- private static void storeMetadataAttribute(CloudBlobWrapper blob,
- String key, String value) {
- HashMap<String, String> metadata = blob.getMetadata();
- if (null == metadata) {
- metadata = new HashMap<String, String>();
- }
- metadata.put(key, value);
- blob.setMetadata(metadata);
- }
-
- private String getMetadataAttribute(HashMap<String, String> metadata,
- String... keyAlternatives) {
- if (null == metadata) {
- return null;
- }
- for (String key : keyAlternatives) {
- if (metadataKeyCaseSensitive) {
- if (metadata.containsKey(key)) {
- return metadata.get(key);
- }
- } else {
- // See HADOOP-17643 for details on why this case insensitive metadata
- // checks been added
- for (Entry<String, String> entry : metadata.entrySet()) {
- if (key.equalsIgnoreCase(entry.getKey())) {
- return entry.getValue();
- }
- }
- }
- }
- return null;
- }
-
- private static void removeMetadataAttribute(CloudBlobWrapper blob,
- String key) {
- HashMap<String, String> metadata = blob.getMetadata();
- if (metadata != null) {
- metadata.remove(key);
- blob.setMetadata(metadata);
- }
- }
-
- private static void storePermissionStatus(CloudBlobWrapper blob,
- PermissionStatus permissionStatus) {
- storeMetadataAttribute(blob, PERMISSION_METADATA_KEY,
- PERMISSION_JSON_SERIALIZER.toJSON(permissionStatus));
- // Remove the old metadata key if present
- removeMetadataAttribute(blob, OLD_PERMISSION_METADATA_KEY);
- }
-
- private PermissionStatus getPermissionStatus(CloudBlobWrapper blob) {
- String permissionMetadataValue = getMetadataAttribute(blob.getMetadata(),
- PERMISSION_METADATA_KEY, OLD_PERMISSION_METADATA_KEY);
- if (permissionMetadataValue != null) {
- return PermissionStatusJsonSerializer.fromJSONString(
- permissionMetadataValue);
- } else {
- return defaultPermissionNoBlobMetadata();
- }
- }
-
- private static void storeFolderAttribute(CloudBlobWrapper blob) {
- storeMetadataAttribute(blob, IS_FOLDER_METADATA_KEY, "true");
- // Remove the old metadata key if present
- removeMetadataAttribute(blob, OLD_IS_FOLDER_METADATA_KEY);
- }
-
- private static String encodeMetadataAttribute(String value) throws UnsupportedEncodingException {
- // We have to URL encode the attribute as it could
- // have URI special characters which unless encoded will result
- // in 403 errors from the server. This is due to metadata properties
- // being sent in the HTTP header of the request which is in turn used
- // on the server side to authorize the request.
- return value == null ? null : URLEncoder.encode(value, METADATA_ENCODING.name());
- }
-
- private static String decodeMetadataAttribute(String encoded) throws UnsupportedEncodingException {
- return encoded == null ? null : URLDecoder.decode(encoded, METADATA_ENCODING.name());
- }
-
- private static String ensureValidAttributeName(String attribute) {
- // Attribute names must be valid C# identifiers so we have to
- // convert the namespace dots (e.g. "user.something") in the
- // attribute names. Using underscores here to be consistent with
- // the constant metadata keys defined earlier in the file
- return attribute.replace('.', '_');
- }
-
- private static void storeLinkAttribute(CloudBlobWrapper blob,
- String linkTarget) throws UnsupportedEncodingException {
- String encodedLinkTarget = encodeMetadataAttribute(linkTarget);
- storeMetadataAttribute(blob,
- LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
- encodedLinkTarget);
- // Remove the old metadata key if present
- removeMetadataAttribute(blob,
- OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
- }
-
- private String getLinkAttributeValue(CloudBlobWrapper blob)
- throws UnsupportedEncodingException {
- String encodedLinkTarget = getMetadataAttribute(blob.getMetadata(),
- LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
- OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
- return decodeMetadataAttribute(encodedLinkTarget);
- }
-
- private boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
- HashMap<String, String> metadata = blob.getMetadata();
- if (null != metadata) {
- if (metadataKeyCaseSensitive) {
- return metadata.containsKey(IS_FOLDER_METADATA_KEY)
- || metadata.containsKey(OLD_IS_FOLDER_METADATA_KEY);
- } else {
- // See HADOOP-17643 for details on why this case insensitive metadata
- // checks been added
- for (String key : metadata.keySet()) {
- if (key.equalsIgnoreCase(IS_FOLDER_METADATA_KEY)
- || key.equalsIgnoreCase(OLD_IS_FOLDER_METADATA_KEY)) {
- return true;
- }
- }
- }
- }
- return false;
- }
-
- private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
- HashMap<String, String> metadata = container.getMetadata();
- if (null == metadata) {
- metadata = new HashMap<String, String>();
- }
- metadata.put(VERSION_METADATA_KEY, CURRENT_WASB_VERSION);
- if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) {
- metadata.remove(OLD_VERSION_METADATA_KEY);
- }
- container.setMetadata(metadata);
- }
-
- private String retrieveVersionAttribute(CloudBlobContainerWrapper container) {
- return getMetadataAttribute(container.getMetadata(), VERSION_METADATA_KEY,
- OLD_VERSION_METADATA_KEY);
- }
-
- @Override
- public void storeEmptyFolder(String key, PermissionStatus permissionStatus)
- throws AzureException {
-
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- // Check if there is an authenticated account associated with the file
- // this instance of the WASB file system. If not the file system has not
- // been authenticated and all access is anonymous.
- if (!isAuthenticatedAccess()) {
- // Preemptively raise an exception indicating no uploads are
- // allowed to anonymous accounts.
- throw new AzureException(
- "Uploads to to public accounts using anonymous access is prohibited.");
- }
-
- try {
- checkContainer(ContainerAccessType.PureWrite);
-
- CloudBlobWrapper blob = getBlobReference(key);
- storePermissionStatus(blob, permissionStatus);
- storeFolderAttribute(blob);
- openOutputStream(blob).close();
- } catch (StorageException e) {
- // Caught exception while attempting upload. Re-throw as an Azure
- // storage exception.
- throw new AzureException(e);
- } catch (URISyntaxException e) {
- throw new AzureException(e);
- } catch (IOException e) {
- Throwable t = e.getCause();
- if (t instanceof StorageException) {
- StorageException se = (StorageException) t;
- // If we got this exception, the blob should have already been created
- if (!"LeaseIdMissing".equals(se.getErrorCode())) {
- throw new AzureException(e);
- }
- } else {
- throw new AzureException(e);
- }
- }
- }
-
- /**
- * Stores an empty blob that's linking to the temporary file where're we're
- * uploading the initial data.
- */
- @Override
- public void storeEmptyLinkFile(String key, String tempBlobKey,
- PermissionStatus permissionStatus) throws AzureException {
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
- // Check if there is an authenticated account associated with the file
- // this instance of the WASB file system. If not the file system has not
- // been authenticated and all access is anonymous.
- if (!isAuthenticatedAccess()) {
- // Preemptively raise an exception indicating no uploads are
- // allowed to anonymous accounts.
- throw new AzureException(
- "Uploads to to public accounts using anonymous access is prohibited.");
- }
-
- try {
- checkContainer(ContainerAccessType.PureWrite);
-
- CloudBlobWrapper blob = getBlobReference(key);
- storePermissionStatus(blob, permissionStatus);
- storeLinkAttribute(blob, tempBlobKey);
- openOutputStream(blob).close();
- } catch (Exception e) {
- // Caught exception while attempting upload. Re-throw as an Azure
- // storage exception.
- throw new AzureException(e);
- }
- }
-
- /**
- * If the blob with the given key exists and has a link in its metadata to a
- * temporary file (see storeEmptyLinkFile), this method returns the key to
- * that temporary file. Otherwise, returns null.
- */
- @Override
- public String getLinkInFileMetadata(String key) throws AzureException {
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- try {
- checkContainer(ContainerAccessType.PureRead);
-
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
- return getLinkAttributeValue(blob);
- } catch (Exception e) {
- // Caught exception while attempting download. Re-throw as an Azure
- // storage exception.
- throw new AzureException(e);
- }
- }
-
- /**
- * Private method to check for authenticated access.
- *
- * @ returns boolean -- true if access is credentialed and authenticated and
- * false otherwise.
- */
- private boolean isAuthenticatedAccess() throws AzureException {
-
- if (isAnonymousCredentials) {
- // Access to this storage account is unauthenticated.
- return false;
- }
- // Access is authenticated.
- return true;
- }
-
- /**
- * This private method uses the root directory or the original container to
- * list blobs under the directory or container depending on whether the
- * original file system object was constructed with a short- or long-form URI.
- * If the root directory is non-null the URI in the file constructor was in
- * the long form.
- *
- * @param includeMetadata
- * if set, the listed items will have their metadata populated
- * already.
- * @param useFlatBlobListing
- * if set the list is flat, otherwise it is hierarchical.
- *
- * @returns blobItems : iterable collection of blob items.
- * @throws URISyntaxException
- *
- */
- private Iterable<ListBlobItem> listRootBlobs(boolean includeMetadata,
- boolean useFlatBlobListing) throws StorageException, URISyntaxException {
- return rootDirectory.listBlobs(
- null,
- useFlatBlobListing,
- includeMetadata
- ? EnumSet.of(BlobListingDetails.METADATA)
- : EnumSet.noneOf(BlobListingDetails.class),
- null,
- getInstrumentedContext());
- }
-
- /**
- * This private method uses the root directory or the original container to
- * list blobs under the directory or container given a specified prefix for
- * the directory depending on whether the original file system object was
- * constructed with a short- or long-form URI. If the root directory is
- * non-null the URI in the file constructor was in the long form.
- *
- * @param aPrefix
- * : string name representing the prefix of containing blobs.
- * @param includeMetadata
- * if set, the listed items will have their metadata populated
- * already.
- * @param useFlatBlobListing
- * if set the list is flat, otherwise it is hierarchical.
- *
- * @returns blobItems : iterable collection of blob items.
- * @throws URISyntaxException
- *
- */
- private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean includeMetadata,
- boolean useFlatBlobListing) throws StorageException, URISyntaxException {
-
- Iterable<ListBlobItem> list = rootDirectory.listBlobs(aPrefix,
- useFlatBlobListing,
- includeMetadata
- ? EnumSet.of(BlobListingDetails.METADATA)
- : EnumSet.noneOf(BlobListingDetails.class),
- null,
- getInstrumentedContext());
- return list;
- }
-
- /**
- * This private method uses the root directory or the original container to
- * list blobs under the directory or container given a specified prefix for
- * the directory depending on whether the original file system object was
- * constructed with a short- or long-form URI. It also uses the specified flat
- * or hierarchical option, listing details options, request options, and
- * operation context.
- *
- * @param aPrefix
- * string name representing the prefix of containing blobs.
- * @param useFlatBlobListing
- * - the list is flat if true, or hierarchical otherwise.
- * @param listingDetails
- * - determine whether snapshots, metadata, committed/uncommitted
- * data
- * @param options
- * - object specifying additional options for the request. null =
- * default options
- * @param opContext
- * - context of the current operation
- * @returns blobItems : iterable collection of blob items.
- * @throws URISyntaxException
- *
- */
- private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean useFlatBlobListing,
- EnumSet<BlobListingDetails> listingDetails, BlobRequestOptions options,
- OperationContext opContext) throws StorageException, URISyntaxException {
-
- CloudBlobDirectoryWrapper directory = this.container.getDirectoryReference(aPrefix);
- return directory.listBlobs(
- null,
- useFlatBlobListing,
- listingDetails,
- options,
- opContext);
- }
-
- /**
- * This private method uses the root directory or the original container to
- * get the block blob reference depending on whether the original file system
- * object was constructed with a short- or long-form URI. If the root
- * directory is non-null the URI in the file constructor was in the long form.
- *
- * @param aKey
- * : a key used to query Azure for the block blob.
- * @returns blob : a reference to the Azure block blob corresponding to the
- * key.
- * @throws URISyntaxException
- *
- */
- private CloudBlobWrapper getBlobReference(String aKey)
- throws StorageException, URISyntaxException {
-
- CloudBlobWrapper blob = null;
- if (isPageBlobKey(aKey)) {
- blob = this.container.getPageBlobReference(aKey);
- } else {
- blob = this.container.getBlockBlobReference(aKey);
- blob.setStreamMinimumReadSizeInBytes(downloadBlockSizeBytes);
- blob.setWriteBlockSizeInBytes(uploadBlockSizeBytes);
- }
-
- return blob;
- }
-
- /**
- * This private method normalizes the key by stripping the container name from
- * the path and returns a path relative to the root directory of the
- * container.
- *
- * @param keyUri
- * - adjust this key to a path relative to the root directory
- *
- * @returns normKey
- */
- private String normalizeKey(URI keyUri) {
- String normKey;
-
- // Strip the container name from the path and return the path
- // relative to the root directory of the container.
- int parts = isStorageEmulator ? 4 : 3;
- normKey = keyUri.getPath().split("/", parts)[(parts - 1)];
-
- // Return the fixed key.
- return normKey;
- }
-
- /**
- * This private method normalizes the key by stripping the container name from
- * the path and returns a path relative to the root directory of the
- * container.
- *
- * @param blob
- * - adjust the key to this blob to a path relative to the root
- * directory
- *
- * @returns normKey
- */
- private String normalizeKey(CloudBlobWrapper blob) {
- return normalizeKey(blob.getUri());
- }
-
- /**
- * This private method normalizes the key by stripping the container name from
- * the path and returns a path relative to the root directory of the
- * container.
- *
- * @param directory
- * - adjust the key to this directory to a path relative to the root
- * directory
- *
- * @returns normKey
- */
- private String normalizeKey(CloudBlobDirectoryWrapper directory) {
- String dirKey = normalizeKey(directory.getUri());
- // Strip the last delimiter
- if (dirKey.endsWith(PATH_DELIMITER)) {
- dirKey = dirKey.substring(0, dirKey.length() - 1);
- }
- return dirKey;
- }
-
- /**
- * Default method to creates a new OperationContext for the Azure Storage
- * operation that has listeners hooked to it that will update the metrics for
- * this file system. This method does not bind to receive send request
- * callbacks by default.
- *
- * @return The OperationContext object to use.
- */
- private OperationContext getInstrumentedContext() {
- // Default is to not bind to receive send callback events.
- return getInstrumentedContext(false);
- }
-
- /**
- * Creates a new OperationContext for the Azure Storage operation that has
- * listeners hooked to it that will update the metrics for this file system.
- *
- * @param bindConcurrentOOBIo
- * - bind to intercept send request call backs to handle OOB I/O.
- *
- * @return The OperationContext object to use.
- */
- private OperationContext getInstrumentedContext(boolean bindConcurrentOOBIo) {
-
- OperationContext operationContext = new OperationContext();
-
- // Set User-Agent
- operationContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() {
- @Override
- public void eventOccurred(SendingRequestEvent eventArg) {
- HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject();
- String userAgentInfo = String.format(Utility.LOCALE_US, "WASB/%s (%s) %s",
- VersionInfo.getVersion(), userAgentId, BaseRequest.getUserAgent());
- connection.setRequestProperty(Constants.HeaderConstants.USER_AGENT, userAgentInfo);
- }
- });
-
- if (selfThrottlingEnabled) {
- SelfThrottlingIntercept.hook(operationContext, selfThrottlingReadFactor,
- selfThrottlingWriteFactor);
- } else if (autoThrottlingEnabled) {
- ClientThrottlingIntercept.hook(operationContext);
- }
-
- if (bandwidthGaugeUpdater != null) {
- //bandwidthGaugeUpdater is null when we config to skip azure metrics
- ResponseReceivedMetricUpdater.hook(
- operationContext,
- instrumentation,
- bandwidthGaugeUpdater);
- }
-
- // Bind operation context to receive send request callbacks on this operation.
- // If reads concurrent to OOB writes are allowed, the interception will reset
- // the conditional header on all Azure blob storage read requests.
- if (bindConcurrentOOBIo) {
- SendRequestIntercept.bind(operationContext);
- }
-
- if (testHookOperationContext != null) {
- operationContext =
- testHookOperationContext.modifyOperationContext(operationContext);
- }
-
- ErrorMetricUpdater.hook(operationContext, instrumentation);
-
- // Return the operation context.
- return operationContext;
- }
-
- @Override
- public FileMetadata retrieveMetadata(String key) throws IOException {
-
- // Attempts to check status may occur before opening any streams so first,
- // check if a session exists, if not create a session with the Azure storage
- // server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- LOG.debug("Retrieving metadata for {}", key);
-
- try {
- if (checkContainer(ContainerAccessType.PureRead) == ContainerState.DoesntExist) {
- // The container doesn't exist, so spare some service calls and just
- // return null now.
- return null;
- }
-
- // Handle the degenerate cases where the key does not exist or the
- // key is a container.
- if (key.equals("/")) {
- // The key refers to root directory of container.
- // Set the modification time for root to zero.
- return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(),
- BlobMaterialization.Implicit, hadoopBlockSize);
- }
-
- CloudBlobWrapper blob = getBlobReference(key);
-
- // Download attributes and return file metadata only if the blob
- // exists.
- if (null != blob && blob.exists(getInstrumentedContext())) {
-
- LOG.debug("Found {} as an explicit blob. Checking if it's a file or folder.", key);
-
- try {
- // The blob exists, so capture the metadata from the blob
- // properties.
- blob.downloadAttributes(getInstrumentedContext());
- BlobProperties properties = blob.getProperties();
-
- if (retrieveFolderAttribute(blob)) {
- LOG.debug("{} is a folder blob.", key);
- return new FileMetadata(key, properties.getLastModified().getTime(),
- getPermissionStatus(blob), BlobMaterialization.Explicit, hadoopBlockSize);
- } else {
-
- LOG.debug("{} is a normal blob.", key);
-
- return new FileMetadata(
- key, // Always return denormalized key with metadata.
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob), hadoopBlockSize);
- }
- } catch(StorageException e){
- if (!NativeAzureFileSystemHelper.isFileNotFoundException(e)) {
- throw e;
- }
- }
- }
-
- // There is no file with that key name, but maybe it is a folder.
- // Query the underlying folder/container to list the blobs stored
- // there under that key.
- //
- Iterable<ListBlobItem> objects =
- listRootBlobs(
- key,
- true,
- EnumSet.of(BlobListingDetails.METADATA),
- null,
- getInstrumentedContext());
-
- // Check if the directory/container has the blob items.
- for (ListBlobItem blobItem : objects) {
- if (blobItem instanceof CloudBlockBlobWrapper
- || blobItem instanceof CloudPageBlobWrapper) {
- LOG.debug("Found blob as a directory-using this file under it to infer its properties {}",
- blobItem.getUri());
-
- blob = (CloudBlobWrapper) blobItem;
- // The key specifies a directory. Create a FileMetadata object which
- // specifies as such.
- BlobProperties properties = blob.getProperties();
-
- return new FileMetadata(key, properties.getLastModified().getTime(),
- getPermissionStatus(blob), BlobMaterialization.Implicit, hadoopBlockSize);
- }
- }
-
- // Return to caller with a null metadata object.
- return null;
-
- } catch (Exception e) {
- // Re-throw the exception as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public byte[] retrieveAttribute(String key, String attribute) throws IOException {
- try {
- checkContainer(ContainerAccessType.PureRead);
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
-
- String value = getMetadataAttribute(blob.getMetadata(),
- ensureValidAttributeName(attribute));
- value = decodeMetadataAttribute(value);
- return value == null ? null : value.getBytes(METADATA_ENCODING);
- } catch (Exception e) {
- throw new AzureException(e);
- }
- }
-
- @Override
- public void storeAttribute(String key, String attribute, byte[] value) throws IOException {
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
-
- String encodedValue = encodeMetadataAttribute(new String(value, METADATA_ENCODING));
- storeMetadataAttribute(blob, ensureValidAttributeName(attribute), encodedValue);
- blob.uploadMetadata(getInstrumentedContext());
- } catch (Exception e) {
- throw new AzureException(e);
- }
- }
-
- @Override
- public InputStream retrieve(String key) throws AzureException, IOException {
- return retrieve(key, 0);
- }
-
- @Override
- public InputStream retrieve(String key, long startByteOffset)
- throws AzureException, IOException {
- return retrieve(key, startByteOffset, Optional.empty());
- }
-
- @Override
- public InputStream retrieve(String key, long startByteOffset,
- Optional<Configuration> options) throws AzureException, IOException {
- try {
- // Check if a session exists, if not create a session with the
- // Azure storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
- checkContainer(ContainerAccessType.PureRead);
-
- InputStream inputStream = openInputStream(getBlobReference(key), options);
- if (startByteOffset > 0) {
- // Skip bytes and ignore return value. This is okay
- // because if you try to skip too far you will be positioned
- // at the end and reads will not return data.
- inputStream.skip(startByteOffset);
- }
- return inputStream;
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public FileMetadata[] list(String prefix, final int maxListingCount,
- final int maxListingDepth) throws IOException {
- return listInternal(prefix, maxListingCount, maxListingDepth);
- }
-
- private FileMetadata[] listInternal(String prefix, final int maxListingCount,
- final int maxListingDepth)
- throws IOException {
- try {
- checkContainer(ContainerAccessType.PureRead);
-
- if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
- prefix += PATH_DELIMITER;
- }
-
- // Enable flat listing option only if depth is unbounded and config
- // KEY_ENABLE_FLAT_LISTING is enabled.
- boolean enableFlatListing = false;
- if (maxListingDepth < 0 && sessionConfiguration.getBoolean(
- KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
- enableFlatListing = true;
- }
-
- Iterable<ListBlobItem> objects;
- if (prefix.equals("/")) {
- objects = listRootBlobs(true, enableFlatListing);
- } else {
- objects = listRootBlobs(prefix, true, enableFlatListing);
- }
-
- HashMap<String, FileMetadata> fileMetadata = new HashMap<>(256);
-
- for (ListBlobItem blobItem : objects) {
- // Check that the maximum listing count is not exhausted.
- //
- if (0 < maxListingCount
- && fileMetadata.size() >= maxListingCount) {
- break;
- }
-
- if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
- String blobKey = null;
- CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
- BlobProperties properties = blob.getProperties();
-
- // Determine format of the blob name depending on whether an absolute
- // path is being used or not.
- blobKey = normalizeKey(blob);
-
- FileMetadata metadata;
- if (retrieveFolderAttribute(blob)) {
- metadata = new FileMetadata(blobKey,
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- BlobMaterialization.Explicit,
- hadoopBlockSize);
- } else {
- metadata = new FileMetadata(
- blobKey,
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- hadoopBlockSize);
- }
- // Add the metadata but remove duplicates. Note that the azure
- // storage java SDK returns two types of entries: CloudBlobWrappter
- // and CloudDirectoryWrapper. In the case where WASB generated the
- // data, there will be an empty blob for each "directory", and we will
- // receive a CloudBlobWrapper. If there are also files within this
- // "directory", we will also receive a CloudDirectoryWrapper. To
- // complicate matters, the data may not be generated by WASB, in
- // which case we may not have an empty blob for each "directory".
- // So, sometimes we receive both a CloudBlobWrapper and a
- // CloudDirectoryWrapper for each directory, and sometimes we receive
- // one or the other but not both. We remove duplicates, but
- // prefer CloudBlobWrapper over CloudDirectoryWrapper.
- // Furthermore, it is very unfortunate that the list results are not
- // ordered, and it is a partial list which uses continuation. So
- // the HashMap is the best structure to remove the duplicates, despite
- // its potential large size.
- fileMetadata.put(blobKey, metadata);
-
- } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
- CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
- // Determine format of directory name depending on whether an absolute
- // path is being used or not.
- //
- String dirKey = normalizeKey(directory);
- // Strip the last /
- if (dirKey.endsWith(PATH_DELIMITER)) {
- dirKey = dirKey.substring(0, dirKey.length() - 1);
- }
-
- // Reached the targeted listing depth. Return metadata for the
- // directory using default permissions.
- //
- // Note: Something smarter should be done about permissions. Maybe
- // inherit the permissions of the first non-directory blob.
- // Also, getting a proper value for last-modified is tricky.
- FileMetadata directoryMetadata = new FileMetadata(dirKey, 0,
- defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit,
- hadoopBlockSize);
-
- // Add the directory metadata to the list only if it's not already
- // there. See earlier note, we prefer CloudBlobWrapper over
- // CloudDirectoryWrapper because it may have additional metadata (
- // properties and ACLs).
- if (!fileMetadata.containsKey(dirKey)) {
- fileMetadata.put(dirKey, directoryMetadata);
- }
-
- if (!enableFlatListing) {
- // Currently at a depth of one, decrement the listing depth for
- // sub-directories.
- buildUpList(directory, fileMetadata, maxListingCount,
- maxListingDepth - 1);
- }
- }
- }
- return fileMetadata.values().toArray(new FileMetadata[fileMetadata.size()]);
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
- //
- throw new AzureException(e);
- }
- }
-
- /**
- * Build up a metadata list of blobs in an Azure blob directory. This method
- * uses a in-order first traversal of blob directory structures to maintain
- * the sorted order of the blob names.
- *
- * @param aCloudBlobDirectory Azure blob directory
- * @param metadataHashMap a map of file metadata objects for each
- * non-directory blob.
- * @param maxListingCount maximum length of the built up list.
- */
- private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory,
- HashMap<String, FileMetadata> metadataHashMap, final int maxListingCount,
- final int maxListingDepth) throws Exception {
-
- // Push the blob directory onto the stack.
- //
- AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack =
- new AzureLinkedStack<Iterator<ListBlobItem>>();
-
- Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null,
- false, EnumSet.of(BlobListingDetails.METADATA), null,
- getInstrumentedContext());
- Iterator<ListBlobItem> blobItemIterator = blobItems.iterator();
-
- if (0 == maxListingDepth || 0 == maxListingCount) {
- // Recurrence depth and listing count are already exhausted. Return
- // immediately.
- return;
- }
-
- // The directory listing depth is unbounded if the maximum listing depth
- // is negative.
- final boolean isUnboundedDepth = (maxListingDepth < 0);
-
- // Reset the current directory listing depth.
- int listingDepth = 1;
-
- // Loop until all directories have been traversed in-order. Loop only
- // the following conditions are satisfied:
- // (1) The stack is not empty, and
- // (2) maxListingCount > 0 implies that the number of items in the
- // metadata list is less than the max listing count.
- while (null != blobItemIterator
- && (maxListingCount <= 0 || metadataHashMap.size() < maxListingCount)) {
- while (blobItemIterator.hasNext()) {
- // Check if the count of items on the list exhausts the maximum
- // listing count.
- //
- if (0 < maxListingCount && metadataHashMap.size() >= maxListingCount) {
- break;
- }
-
- ListBlobItem blobItem = blobItemIterator.next();
-
- // Add the file metadata to the list if this is not a blob
- // directory item.
- //
- if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
- String blobKey = null;
- CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
- BlobProperties properties = blob.getProperties();
-
- // Determine format of the blob name depending on whether an absolute
- // path is being used or not.
- blobKey = normalizeKey(blob);
-
- FileMetadata metadata;
- if (retrieveFolderAttribute(blob)) {
- metadata = new FileMetadata(blobKey,
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- BlobMaterialization.Explicit,
- hadoopBlockSize);
- } else {
- metadata = new FileMetadata(
- blobKey,
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- hadoopBlockSize);
- }
-
- // Add the metadata but remove duplicates. Note that the azure
- // storage java SDK returns two types of entries: CloudBlobWrappter
- // and CloudDirectoryWrapper. In the case where WASB generated the
- // data, there will be an empty blob for each "directory", and we will
- // receive a CloudBlobWrapper. If there are also files within this
- // "directory", we will also receive a CloudDirectoryWrapper. To
- // complicate matters, the data may not be generated by WASB, in
- // which case we may not have an empty blob for each "directory".
- // So, sometimes we receive both a CloudBlobWrapper and a
- // CloudDirectoryWrapper for each directory, and sometimes we receive
- // one or the other but not both. We remove duplicates, but
- // prefer CloudBlobWrapper over CloudDirectoryWrapper.
- // Furthermore, it is very unfortunate that the list results are not
- // ordered, and it is a partial list which uses continuation. So
- // the HashMap is the best structure to remove the duplicates, despite
- // its potential large size.
- metadataHashMap.put(blobKey, metadata);
- } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
- CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
-
- // This is a directory blob, push the current iterator onto
- // the stack of iterators and start iterating through the current
- // directory.
- if (isUnboundedDepth || maxListingDepth > listingDepth) {
- // Push the current directory on the stack and increment the listing
- // depth.
- dirIteratorStack.push(blobItemIterator);
- ++listingDepth;
-
- // The current blob item represents the new directory. Get
- // an iterator for this directory and continue by iterating through
- // this directory.
- blobItems = directory.listBlobs(null, false,
- EnumSet.noneOf(BlobListingDetails.class), null,
- getInstrumentedContext());
- blobItemIterator = blobItems.iterator();
- } else {
- // Determine format of directory name depending on whether an
- // absolute path is being used or not.
- String dirKey = normalizeKey(directory);
-
- // Add the directory metadata to the list only if it's not already
- // there. See earlier note, we prefer CloudBlobWrapper over
- // CloudDirectoryWrapper because it may have additional metadata (
- // properties and ACLs).
- if (!metadataHashMap.containsKey(dirKey)) {
-
- // Reached the targeted listing depth. Return metadata for the
- // directory using default permissions.
- //
- // Note: Something smarter should be done about permissions. Maybe
- // inherit the permissions of the first non-directory blob.
- // Also, getting a proper value for last-modified is tricky.
- //
- FileMetadata directoryMetadata = new FileMetadata(dirKey,
- 0,
- defaultPermissionNoBlobMetadata(),
- BlobMaterialization.Implicit,
- hadoopBlockSize);
-
- // Add the directory metadata to the list.
- metadataHashMap.put(dirKey, directoryMetadata);
- }
- }
- }
- }
-
- // Traversal of directory tree
-
- // Check if the iterator stack is empty. If it is set the next blob
- // iterator to null. This will act as a terminator for the for-loop.
- // Otherwise pop the next iterator from the stack and continue looping.
- //
- if (dirIteratorStack.isEmpty()) {
- blobItemIterator = null;
- } else {
- // Pop the next directory item from the stack and decrement the
- // depth.
- blobItemIterator = dirIteratorStack.pop();
- --listingDepth;
-
- // Assertion: Listing depth should not be less than zero.
- if (listingDepth < 0) {
- throw new AssertionError("Non-negative listing depth expected");
- }
- }
- }
- }
-
- /**
- * Return the actual data length of the blob with the specified properties.
- * If it is a page blob, you can't rely on the length from the properties
- * argument and you must get it from the file. Otherwise, you can.
- */
- private long getDataLength(CloudBlobWrapper blob, BlobProperties properties)
- throws AzureException {
- if (blob instanceof CloudPageBlobWrapper) {
- try {
- return PageBlobInputStream.getPageBlobDataSize((CloudPageBlobWrapper) blob,
- getInstrumentedContext(
- isConcurrentOOBAppendAllowed()));
- } catch (Exception e) {
- throw new AzureException(
- "Unexpected exception getting page blob actual data size.", e);
- }
- }
- return properties.getLength();
- }
-
- /**
- * Deletes the given blob, taking special care that if we get a
- * blob-not-found exception upon retrying the operation, we just
- * swallow the error since what most probably happened is that
- * the first operation succeeded on the server.
- * @param blob The blob to delete.
- * @param lease Azure blob lease, or null if no lease is to be used.
- * @throws StorageException
- */
- private void safeDelete(CloudBlobWrapper blob, SelfRenewingLease lease) throws StorageException {
- OperationContext operationContext = getInstrumentedContext();
- try {
- blob.delete(operationContext, lease);
- } catch (StorageException e) {
- if (!NativeAzureFileSystemHelper.isFileNotFoundException(e)) {
- LOG.error("Encountered Storage Exception for delete on Blob: {}"
- + ", Exception Details: {} Error Code: {}",
- blob.getUri(), e.getMessage(), e.getErrorCode());
- }
- // On exception, check that if:
- // 1. It's a BlobNotFound exception AND
- // 2. It got there after one-or-more retries THEN
- // we swallow the exception.
- if (e.getErrorCode() != null
- && "BlobNotFound".equals(e.getErrorCode())
- && operationContext.getRequestResults().size() > 1
- && operationContext.getRequestResults().get(0).getException() != null) {
- LOG.debug("Swallowing delete exception on retry: {}", e.getMessage());
- return;
- } else {
- throw e;
- }
- } finally {
- if (lease != null) {
- lease.free();
- }
- }
- }
-
- /**
- * API implementation to delete a blob in the back end azure storage.
- */
- @Override
- public boolean delete(String key, SelfRenewingLease lease) throws IOException {
- try {
- if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) {
- // Container doesn't exist, no need to do anything
- return true;
- }
- // Get the blob reference and delete it.
- CloudBlobWrapper blob = getBlobReference(key);
- safeDelete(blob, lease);
- return true;
- } catch (Exception e) {
- if (e instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException(
- (StorageException) e)) {
- // the file or directory does not exist
- return false;
- }
- throw new AzureException(e);
- }
- }
-
- /**
- * API implementation to delete a blob in the back end azure storage.
- */
- @Override
- public boolean delete(String key) throws IOException {
- try {
- return delete(key, null);
- } catch (IOException e) {
- Throwable t = e.getCause();
- if (t instanceof StorageException) {
- StorageException se = (StorageException) t;
- if ("LeaseIdMissing".equals(se.getErrorCode())){
- SelfRenewingLease lease = null;
- try {
- lease = acquireLease(key);
- return delete(key, lease);
- } catch (AzureException e3) {
- LOG.warn("Got unexpected exception trying to acquire lease on "
- + key + "." + e3.getMessage());
- throw e3;
- } finally {
- try {
- if (lease != null){
- lease.free();
- }
- } catch (Exception e4){
- LOG.error("Unable to free lease on " + key, e4);
- }
- }
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- }
- }
-
- @Override
- public void rename(String srcKey, String dstKey) throws IOException {
- rename(srcKey, dstKey, false, null, true);
- }
-
- @Override
- public void rename(String srcKey, String dstKey, boolean acquireLease,
- SelfRenewingLease existingLease) throws IOException {
- rename(srcKey, dstKey, acquireLease, existingLease, true);
- }
-
- @Override
- public void rename(String srcKey, String dstKey, boolean acquireLease,
- SelfRenewingLease existingLease, boolean overwriteDestination) throws IOException {
-
- LOG.debug("Moving {} to {}", srcKey, dstKey);
-
- if (acquireLease && existingLease != null) {
- throw new IOException("Cannot acquire new lease if one already exists.");
- }
-
- CloudBlobWrapper srcBlob = null;
- CloudBlobWrapper dstBlob = null;
- SelfRenewingLease lease = null;
- try {
- // Attempts rename may occur before opening any streams so first,
- // check if a session exists, if not create a session with the Azure
- // storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- checkContainer(ContainerAccessType.ReadThenWrite);
- // Get the source blob and assert its existence. If the source key
- // needs to be normalized then normalize it.
- //
-
- srcBlob = getBlobReference(srcKey);
- if (!srcBlob.exists(getInstrumentedContext())) {
- throw new AzureException("Source blob " + srcKey + " does not exist.");
- }
-
- /**
- * Conditionally get a lease on the source blob to prevent other writers
- * from changing it. This is used for correctness in HBase when log files
- * are renamed. It generally should do no harm other than take a little
- * more time for other rename scenarios. When the HBase master renames a
- * log file folder, the lease locks out other writers. This
- * prevents a region server that the master thinks is dead, but is still
- * alive, from committing additional updates. This is different than
- * when HBase runs on HDFS, where the region server recovers the lease
- * on a log file, to gain exclusive access to it, before it splits it.
- */
- if (acquireLease) {
- lease = srcBlob.acquireLease();
- } else if (existingLease != null) {
- lease = existingLease;
- }
-
- // Get the destination blob. The destination key always needs to be
- // normalized.
- //
- dstBlob = getBlobReference(dstKey);
-
- // Rename the source blob to the destination blob by copying it to
- // the destination blob then deleting it.
- //
- // Copy blob operation in Azure storage is very costly. It will be highly
- // likely throttled during Azure storage gc. Short term fix will be using
- // a more intensive exponential retry policy when the cluster is getting
- // throttled.
- try {
- dstBlob.startCopyFromBlob(srcBlob, null,
- getInstrumentedContext(), overwriteDestination);
- } catch (StorageException se) {
- if (se.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
- int copyBlobMinBackoff = sessionConfiguration.getInt(
- KEY_COPYBLOB_MIN_BACKOFF_INTERVAL,
- DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL);
-
- int copyBlobMaxBackoff = sessionConfiguration.getInt(
- KEY_COPYBLOB_MAX_BACKOFF_INTERVAL,
- DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL);
-
- int copyBlobDeltaBackoff = sessionConfiguration.getInt(
- KEY_COPYBLOB_BACKOFF_INTERVAL,
- DEFAULT_COPYBLOB_BACKOFF_INTERVAL);
-
- int copyBlobMaxRetries = sessionConfiguration.getInt(
- KEY_COPYBLOB_MAX_IO_RETRIES,
- DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS);
-
- BlobRequestOptions options = new BlobRequestOptions();
- options.setRetryPolicyFactory(new RetryExponentialRetry(
- copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff,
- copyBlobMaxRetries));
- dstBlob.startCopyFromBlob(srcBlob, options,
- getInstrumentedContext(), overwriteDestination);
- } else {
- throw se;
- }
- }
- waitForCopyToComplete(dstBlob, getInstrumentedContext());
- safeDelete(srcBlob, lease);
- } catch (StorageException e) {
- if (e.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
- LOG.warn("Rename: CopyBlob: StorageException: ServerBusy: Retry complete, will attempt client side copy for page blob");
- InputStream ipStream = null;
- OutputStream opStream = null;
- try {
- if (srcBlob.getProperties().getBlobType() == BlobType.PAGE_BLOB){
- ipStream = openInputStream(srcBlob, Optional.empty());
- opStream = openOutputStream(dstBlob);
- byte[] buffer = new byte[PageBlobFormatHelpers.PAGE_SIZE];
- int len;
- while ((len = ipStream.read(buffer)) != -1) {
- opStream.write(buffer, 0, len);
- }
- opStream.flush();
- opStream.close();
- ipStream.close();
- } else {
- throw new AzureException(e);
- }
- safeDelete(srcBlob, lease);
- } catch(StorageException se) {
- LOG.warn("Rename: CopyBlob: StorageException: Failed");
- throw new AzureException(se);
- } finally {
- IOUtils.closeStream(ipStream);
- IOUtils.closeStream(opStream);
- }
- } else {
- throw new AzureException(e);
- }
- } catch (URISyntaxException e) {
- // Re-throw exception as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- private void waitForCopyToComplete(CloudBlobWrapper blob, OperationContext opContext){
- boolean copyInProgress = true;
- while (copyInProgress) {
- try {
- blob.downloadAttributes(opContext);
- }
- catch (StorageException se){
- }
-
- // test for null because mocked filesystem doesn't know about copystates yet.
- copyInProgress = (blob.getCopyState() != null && blob.getCopyState().getStatus() == CopyStatus.PENDING);
- if (copyInProgress) {
- try {
- Thread.sleep(1000);
- }
- catch (InterruptedException ie){
- //ignore
- }
- }
- }
- }
-
- /**
- * Checks whether an explicit file/folder exists.
- * This is used by redo of atomic rename.
- * There was a bug(apache jira HADOOP-12780) during atomic rename if
- * process crashes after an inner directory has been renamed but still
- * there are file under that directory to be renamed then after the
- * process comes again it tries to redo the renames. It checks whether
- * the directory exists or not by calling filesystem.exist.
- * But filesystem.Exists will treat that directory as implicit directory
- * and return true as file exists under that directory. So It will try
- * try to rename that directory and will fail as the corresponding blob
- * does not exist. So this method explicitly checks for the blob.
- */
- @Override
- public boolean explicitFileExists(String key) throws AzureException {
- CloudBlobWrapper blob;
- try {
- blob = getBlobReference(key);
- if (null != blob && blob.exists(getInstrumentedContext())) {
- return true;
- }
-
- return false;
- } catch (StorageException e) {
- throw new AzureException(e);
- } catch (URISyntaxException e) {
- throw new AzureException(e);
- }
- }
-
- /**
- * Changes the permission status on the given key.
- */
- @Override
- public void changePermissionStatus(String key, PermissionStatus newPermission)
- throws AzureException {
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
- storePermissionStatus(blob, newPermission);
- blob.uploadMetadata(getInstrumentedContext());
- } catch (Exception e) {
- throw new AzureException(e);
- }
- }
-
- @Override
- public void purge(String prefix) throws IOException {
- try {
-
- // Attempts to purge may occur before opening any streams so first,
- // check if a session exists, if not create a session with the Azure
- // storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) {
- // Container doesn't exist, no need to do anything.
- return;
- }
- // Get all blob items with the given prefix from the container and delete
- // them.
- Iterable<ListBlobItem> objects = listRootBlobs(prefix, false, false);
- for (ListBlobItem blobItem : objects) {
- ((CloudBlob) blobItem).delete(DeleteSnapshotsOption.NONE, null, null,
- getInstrumentedContext());
- }
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
- //
- throw new AzureException(e);
- }
- }
-
- /**
- * Get a lease on the blob identified by key. This lease will be renewed
- * indefinitely by a background thread.
- */
- @Override
- public SelfRenewingLease acquireLease(String key) throws AzureException {
- LOG.debug("acquiring lease on {}", key);
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- return blob.acquireLease();
- }
- catch (Exception e) {
-
- // Caught exception while attempting to get lease. Re-throw as an
- // Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public void updateFolderLastModifiedTime(String key, Date lastModified,
- SelfRenewingLease folderLease)
- throws AzureException {
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified
- //timestamp to current time
- blob.uploadProperties(getInstrumentedContext(), folderLease);
- } catch (Exception e) {
-
- // Caught exception while attempting to update the properties. Re-throw as an
- // Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public void updateFolderLastModifiedTime(String key,
- SelfRenewingLease folderLease) throws AzureException {
- final Calendar lastModifiedCalendar = Calendar
- .getInstance(Utility.LOCALE_US);
- lastModifiedCalendar.setTimeZone(Utility.UTC_ZONE);
- Date lastModified = lastModifiedCalendar.getTime();
- updateFolderLastModifiedTime(key, lastModified, folderLease);
- }
-
- @Override
- public void dump() throws IOException {
- }
-
- @Override
- public void close() {
- if (bandwidthGaugeUpdater != null) {
- bandwidthGaugeUpdater.close();
- bandwidthGaugeUpdater = null;
- }
- }
-
- // Finalizer to ensure complete shutdown
- @Override
- protected void finalize() throws Throwable {
- LOG.debug("finalize() called");
- close();
- super.finalize();
- }
-
- @Override
- public DataOutputStream retrieveAppendStream(String key, int bufferSize) throws IOException {
-
- try {
-
- if (isPageBlobKey(key)) {
- throw new UnsupportedOperationException("Append not supported for Page Blobs");
- }
-
- CloudBlobWrapper blob = this.container.getBlockBlobReference(key);
-
- OutputStream outputStream;
-
- BlockBlobAppendStream blockBlobOutputStream = new BlockBlobAppendStream(
- (CloudBlockBlobWrapper) blob,
- key,
- bufferSize,
- isBlockBlobWithCompactionKey(key),
- getInstrumentedContext());
-
- outputStream = blockBlobOutputStream;
-
- DataOutputStream dataOutStream = new SyncableDataOutputStream(
- outputStream);
-
- return dataOutStream;
- } catch(Exception ex) {
- throw new AzureException(ex);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java
deleted file mode 100644
index a1f82428b49..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Indicates whether there are actual blobs indicating the existence of
- * directories or whether we're inferring their existence from them having files
- * in there.
- */
-@InterfaceAudience.Private
-enum BlobMaterialization {
- /**
- * Indicates a directory that isn't backed by an actual blob, but its
- * existence is implied by the fact that there are files in there. For
- * example, if the blob /a/b exists then it implies the existence of the /a
- * directory if there's no /a blob indicating it.
- */
- Implicit,
- /**
- * Indicates that the directory is backed by an actual blob that has the
- * isFolder metadata on it.
- */
- Explicit,
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java
deleted file mode 100644
index 6da64e124ef..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import com.microsoft.azure.storage.Constants.HeaderConstants;
-import org.apache.hadoop.classification.InterfaceAudience;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Determines the operation type (PutBlock, PutPage, GetBlob, etc) of Azure
- * Storage operations. This is used by the handlers of the SendingRequestEvent
- * and ResponseReceivedEvent exposed by the Azure Storage SDK to identify
- * operation types (since the type of operation is not exposed by the SDK).
- */
-@InterfaceAudience.Private
-final class BlobOperationDescriptor {
-
- private BlobOperationDescriptor() {
- // hide default constructor
- }
-
- /**
- * Gets the content length for the Azure Storage operation from the
- * 'x-ms-range' header, if set.
- * @param range the value of the 'x-ms-range' header.
- * @return the content length, or zero if not set.
- */
- private static long getContentLengthIfKnown(String range) {
- long contentLength = 0;
- // Format is "bytes=%d-%d"
- if (range != null && range.startsWith("bytes=")) {
- String[] offsets = range.substring("bytes=".length()).split("-");
- if (offsets.length == 2) {
- contentLength = Long.parseLong(offsets[1]) - Long.parseLong(offsets[0])
- + 1;
- }
- }
- return contentLength;
- }
-
- /**
- * Gets the content length for the Azure Storage operation, or returns zero if
- * unknown.
- * @param conn the connection object for the Azure Storage operation.
- * @param operationType the Azure Storage operation type.
- * @return the content length, or zero if unknown.
- */
- static long getContentLengthIfKnown(HttpURLConnection conn,
- OperationType operationType) {
- long contentLength = 0;
- switch (operationType) {
- case AppendBlock:
- case PutBlock:
- String lengthString = conn.getRequestProperty(
- HeaderConstants.CONTENT_LENGTH);
- contentLength = (lengthString != null)
- ? Long.parseLong(lengthString)
- : 0;
- break;
- case PutPage:
- case GetBlob:
- contentLength = BlobOperationDescriptor.getContentLengthIfKnown(
- conn.getRequestProperty("x-ms-range"));
- break;
- default:
- break;
- }
- return contentLength;
- }
-
- /**
- * Gets the operation type of an Azure Storage operation.
- *
- * @param conn the connection object for the Azure Storage operation.
- * @return the operation type.
- */
- static OperationType getOperationType(HttpURLConnection conn) {
- OperationType operationType = OperationType.Unknown;
- String method = conn.getRequestMethod();
- String compValue = getQueryParameter(conn.getURL(),
- "comp");
-
- if (method.equalsIgnoreCase("PUT")) {
- if (compValue != null) {
- switch (compValue) {
- case "metadata":
- operationType = OperationType.SetMetadata;
- break;
- case "properties":
- operationType = OperationType.SetProperties;
- break;
- case "block":
- operationType = OperationType.PutBlock;
- break;
- case "page":
- String pageWrite = conn.getRequestProperty("x-ms-page-write");
- if (pageWrite != null && pageWrite.equalsIgnoreCase(
- "UPDATE")) {
- operationType = OperationType.PutPage;
- }
- break;
- case "appendblock":
- operationType = OperationType.AppendBlock;
- break;
- case "blocklist":
- operationType = OperationType.PutBlockList;
- break;
- default:
- break;
- }
- } else {
- String blobType = conn.getRequestProperty("x-ms-blob-type");
- if (blobType != null
- && (blobType.equalsIgnoreCase("PageBlob")
- || blobType.equalsIgnoreCase("BlockBlob")
- || blobType.equalsIgnoreCase("AppendBlob"))) {
- operationType = OperationType.CreateBlob;
- } else if (blobType == null) {
- String resType = getQueryParameter(conn.getURL(),
- "restype");
- if (resType != null
- && resType.equalsIgnoreCase("container")) {
- operationType = operationType.CreateContainer;
- }
- }
- }
- } else if (method.equalsIgnoreCase("GET")) {
- if (compValue != null) {
- switch (compValue) {
- case "list":
- operationType = OperationType.ListBlobs;
- break;
-
- case "metadata":
- operationType = OperationType.GetMetadata;
- break;
- case "blocklist":
- operationType = OperationType.GetBlockList;
- break;
- case "pagelist":
- operationType = OperationType.GetPageList;
- break;
- default:
- break;
- }
- } else if (conn.getRequestProperty("x-ms-range") != null) {
- operationType = OperationType.GetBlob;
- }
- } else if (method.equalsIgnoreCase("HEAD")) {
- operationType = OperationType.GetProperties;
- } else if (method.equalsIgnoreCase("DELETE")) {
- String resType = getQueryParameter(conn.getURL(),
- "restype");
- if (resType != null
- && resType.equalsIgnoreCase("container")) {
- operationType = operationType.DeleteContainer;
- } else {
- operationType = OperationType.DeleteBlob;
- }
- }
- return operationType;
- }
-
- private static String getQueryParameter(URL url, String queryParameterName) {
- String query = (url != null) ? url.getQuery(): null;
-
- if (query == null) {
- return null;
- }
-
- String searchValue = queryParameterName + "=";
-
- int offset = query.indexOf(searchValue);
- String value = null;
- if (offset != -1) {
- int beginIndex = offset + searchValue.length();
- int endIndex = query.indexOf('&', beginIndex);
- value = (endIndex == -1)
- ? query.substring(beginIndex)
- : query.substring(beginIndex, endIndex);
- }
- return value;
- }
-
- @InterfaceAudience.Private
- enum OperationType {
- AppendBlock,
- CreateBlob,
- CreateContainer,
- DeleteBlob,
- DeleteContainer,
- GetBlob,
- GetBlockList,
- GetMetadata,
- GetPageList,
- GetProperties,
- ListBlobs,
- PutBlock,
- PutBlockList,
- PutPage,
- SetMetadata,
- SetProperties,
- Unknown
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
deleted file mode 100644
index c19bbe48fc3..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ /dev/null
@@ -1,1141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-import java.util.Random;
-import java.util.concurrent.ConcurrentLinkedDeque;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.util.Preconditions;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-import org.apache.hadoop.io.ElasticByteBufferPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.StorageErrorCodeStrings;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.BlockSearchMode;
-
-/**
- * Stream object that implements append for Block Blobs in WASB.
- *
- * The stream object implements hflush/hsync and block compaction. Block
- * compaction is the process of replacing a sequence of small blocks with one
- * big block. Azure Block blobs supports up to 50000 blocks and every
- * hflush/hsync generates one block. When the number of blocks is above 32000,
- * the process of compaction decreases the total number of blocks, if possible.
- * If compaction is disabled, hflush/hsync are empty functions.
- *
- * The stream object uses background threads for uploading the blocks and the
- * block blob list. Blocks can be uploaded concurrently. However, when the block
- * list is uploaded, block uploading should stop. If a block is uploaded before
- * the block list and the block id is not in the list, the block will be lost.
- * If the block is uploaded after the block list and the block id is in the
- * list, the block list upload will fail. The exclusive access for the block
- * list upload is managed by uploadingSemaphore.
- */
-
-public class BlockBlobAppendStream extends OutputStream implements Syncable,
- StreamCapabilities {
-
- /**
- * The name of the blob/file.
- */
- private final String key;
-
- /**
- * This variable tracks if this is new blob or existing one.
- */
- private boolean blobExist;
-
- /**
- * When the blob exist, to to prevent concurrent write we take a lease.
- * Taking a lease is not necessary for new blobs.
- */
- private SelfRenewingLease lease = null;
-
- /**
- * The support for process of compaction is optional.
- */
- private final boolean compactionEnabled;
-
- /**
- * The number of blocks above each block compaction is triggered.
- */
- private static final int DEFAULT_ACTIVATE_COMPACTION_BLOCK_COUNT = 32000;
-
- /**
- * The number of blocks above each block compaction is triggered.
- */
- private int activateCompactionBlockCount
- = DEFAULT_ACTIVATE_COMPACTION_BLOCK_COUNT;
-
- /**
- * The size of the output buffer. Writes store the data in outBuffer until
- * either the size is above maxBlockSize or hflush/hsync is called.
- */
- private final AtomicInteger maxBlockSize;
-
- /**
- * The current buffer where writes are stored.
- */
- private ByteBuffer outBuffer;
-
- /**
- * The size of the blob that has been successfully stored in the Azure Blob
- * service.
- */
- private final AtomicLong committedBlobLength = new AtomicLong(0);
-
- /**
- * Position of last block in the blob.
- */
- private volatile long blobLength = 0;
-
- /**
- * Minutes waiting before the close operation timed out.
- */
- private static final int CLOSE_UPLOAD_DELAY = 10;
-
- /**
- * Keep alive time for the threadpool.
- */
- private static final int THREADPOOL_KEEP_ALIVE = 30;
- /**
- * Azure Block Blob used for the stream.
- */
- private final CloudBlockBlobWrapper blob;
-
- /**
- * Azure Storage operation context.
- */
- private final OperationContext opContext;
-
- /**
- * Commands send from client calls to the background thread pool.
- */
- private abstract class UploadCommand {
-
- // the blob offset for the command
- private final long commandBlobOffset;
-
- // command completion latch
- private final CountDownLatch completed = new CountDownLatch(1);
-
- UploadCommand(long offset) {
- this.commandBlobOffset = offset;
- }
-
- long getCommandBlobOffset() {
- return commandBlobOffset;
- }
-
- void await() throws InterruptedException {
- completed.await();
- }
-
- void awaitAsDependent() throws InterruptedException {
- await();
- }
-
- void setCompleted() {
- completed.countDown();
- }
-
- void execute() throws InterruptedException, IOException {}
-
- void dump() {}
- }
-
- /**
- * The list of recent commands. Before block list is committed, all the block
- * listed in the list must be uploaded. activeBlockCommands is used for
- * enumerating the blocks and waiting on the latch until the block is
- * uploaded.
- */
- private final ConcurrentLinkedQueue<UploadCommand> activeBlockCommands
- = new ConcurrentLinkedQueue<>();
-
- /**
- * Variable to track if the stream has been closed.
- */
- private volatile boolean closed = false;
-
- /**
- * First IOException encountered.
- */
- private final AtomicReference<IOException> firstError
- = new AtomicReference<>();
-
- /**
- * Flag set when the first error has been thrown.
- */
- private boolean firstErrorThrown = false;
-
- /**
- * Semaphore for serializing block uploads with NativeAzureFileSystem.
- *
- * The semaphore starts with number of permits equal to the number of block
- * upload threads. Each block upload thread needs one permit to start the
- * upload. The put block list acquires all the permits before the block list
- * is committed.
- */
- private final Semaphore uploadingSemaphore = new Semaphore(
- MAX_NUMBER_THREADS_IN_THREAD_POOL,
- true);
-
- /**
- * Queue storing buffers with the size of the Azure block ready for
- * reuse. The pool allows reusing the blocks instead of allocating new
- * blocks. After the data is sent to the service, the buffer is returned
- * back to the queue
- */
- private final ElasticByteBufferPool poolReadyByteBuffers
- = new ElasticByteBufferPool();
-
- /**
- * The blob's block list.
- */
- private final List<BlockEntry> blockEntries = new ArrayList<>(
- DEFAULT_CAPACITY_BLOCK_ENTRIES);
-
- private static final int DEFAULT_CAPACITY_BLOCK_ENTRIES = 1024;
-
- /**
- * The uncommitted blob's block list.
- */
- private final ConcurrentLinkedDeque<BlockEntry> uncommittedBlockEntries
- = new ConcurrentLinkedDeque<>();
-
- /**
- * Variable to hold the next block id to be used for azure storage blocks.
- */
- private static final int UNSET_BLOCKS_COUNT = -1;
- private long nextBlockCount = UNSET_BLOCKS_COUNT;
-
- /**
- * Variable to hold the block id prefix to be used for azure storage blocks.
- */
- private String blockIdPrefix = null;
-
- /**
- * Maximum number of threads in block upload thread pool.
- */
- private static final int MAX_NUMBER_THREADS_IN_THREAD_POOL = 4;
-
- /**
- * Number of times block upload needs is retried.
- */
- private static final int MAX_BLOCK_UPLOAD_RETRIES = 3;
-
- /**
- * Wait time between block upload retries in milliseconds.
- */
- private static final int BLOCK_UPLOAD_RETRY_INTERVAL = 1000;
-
- /**
- * Logger.
- */
- private static final Logger LOG =
- LoggerFactory.getLogger(BlockBlobAppendStream.class);
-
- /**
- * The absolute maximum of blocks for a blob. It includes committed and
- * temporary blocks.
- */
- private static final int MAX_BLOCK_COUNT = 100000;
-
- /**
- * The upload thread pool executor.
- */
- private ThreadPoolExecutor ioThreadPool;
-
- /**
- * Azure Storage access conditions for the blob.
- */
- private final AccessCondition accessCondition = new AccessCondition();
-
- /**
- * Atomic integer to provide thread id for thread names for uploader threads.
- */
- private final AtomicInteger threadSequenceNumber;
-
- /**
- * Prefix to be used for thread names for uploader threads.
- */
- private static final String THREAD_ID_PREFIX = "append-blockblob";
-
- /**
- * BlockBlobAppendStream constructor.
- *
- * @param blob
- * Azure Block Blob
- * @param aKey
- * blob's name
- * @param bufferSize
- * the maximum size of a blob block.
- * @param compactionEnabled
- * is the compaction process enabled for this blob
- * @param opContext
- * Azure Store operation context for the blob
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream cannot be used for append operations
- */
- public BlockBlobAppendStream(final CloudBlockBlobWrapper blob,
- final String aKey,
- final int bufferSize,
- final boolean compactionEnabled,
- final OperationContext opContext)
- throws IOException {
-
- Preconditions.checkArgument(StringUtils.isNotEmpty(aKey));
- Preconditions.checkArgument(bufferSize >= 0);
-
- this.blob = blob;
- this.opContext = opContext;
- this.key = aKey;
- this.maxBlockSize = new AtomicInteger(bufferSize);
- this.threadSequenceNumber = new AtomicInteger(0);
- this.blockIdPrefix = null;
- this.compactionEnabled = compactionEnabled;
- this.blobExist = true;
- this.outBuffer = poolReadyByteBuffers.getBuffer(false, maxBlockSize.get());
-
- try {
- // download the block list
- blockEntries.addAll(
- blob.downloadBlockList(
- BlockListingFilter.COMMITTED,
- new BlobRequestOptions(),
- opContext));
-
- blobLength = blob.getProperties().getLength();
-
- committedBlobLength.set(blobLength);
-
- // Acquiring lease on the blob.
- lease = new SelfRenewingLease(blob, true);
- accessCondition.setLeaseID(lease.getLeaseID());
-
- } catch (StorageException ex) {
- if (ex.getErrorCode().equals(StorageErrorCodeStrings.BLOB_NOT_FOUND)) {
- blobExist = false;
- }
- else if (ex.getErrorCode().equals(
- StorageErrorCodeStrings.LEASE_ALREADY_PRESENT)) {
- throw new AzureException(
- "Unable to set Append lease on the Blob: " + ex, ex);
- }
- else {
- LOG.debug(
- "Encountered storage exception."
- + " StorageException : {} ErrorCode : {}",
- ex,
- ex.getErrorCode());
- throw new AzureException(ex);
- }
- }
-
- setBlocksCountAndBlockIdPrefix(blockEntries);
-
- this.ioThreadPool = new ThreadPoolExecutor(
- MAX_NUMBER_THREADS_IN_THREAD_POOL,
- MAX_NUMBER_THREADS_IN_THREAD_POOL,
- THREADPOOL_KEEP_ALIVE,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(),
- new UploaderThreadFactory());
- }
-
- /**
- * Set payload size of the stream.
- * It is intended to be used for unit testing purposes only.
- */
- @VisibleForTesting
- synchronized void setMaxBlockSize(int size) {
- maxBlockSize.set(size);
-
- // it is for testing only so we can abandon the previously allocated
- // payload
- this.outBuffer = ByteBuffer.allocate(maxBlockSize.get());
- }
-
- /**
- * Set compaction parameters.
- * It is intended to be used for unit testing purposes only.
- */
- @VisibleForTesting
- void setCompactionBlockCount(int activationCount) {
- activateCompactionBlockCount = activationCount;
- }
-
- /**
- * Get the list of block entries. It is used for testing purposes only.
- * @return List of block entries.
- */
- @VisibleForTesting
- List<BlockEntry> getBlockList() throws StorageException, IOException {
- return blob.downloadBlockList(
- BlockListingFilter.COMMITTED,
- new BlobRequestOptions(),
- opContext);
- }
-
- /**
- * Writes the specified byte to this output stream. The general contract for
- * write is that one byte is written to the output stream. The byte to be
- * written is the eight low-order bits of the argument b. The 24 high-order
- * bits of b are ignored.
- *
- * @param byteVal
- * the byteValue to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public void write(final int byteVal) throws IOException {
- write(new byte[] { (byte) (byteVal & 0xFF) });
- }
-
- /**
- * Writes length bytes from the specified byte array starting at offset to
- * this output stream.
- *
- * @param data
- * the byte array to write.
- * @param offset
- * the start offset in the data.
- * @param length
- * the number of bytes to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public synchronized void write(final byte[] data, int offset, int length)
- throws IOException {
- Preconditions.checkArgument(data != null, "null data");
-
- if (offset < 0 || length < 0 || length > data.length - offset) {
- throw new IndexOutOfBoundsException();
- }
-
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
-
- while (outBuffer.remaining() < length) {
-
- int remaining = outBuffer.remaining();
- outBuffer.put(data, offset, remaining);
-
- // upload payload to azure storage
- addBlockUploadCommand();
-
- offset += remaining;
- length -= remaining;
- }
-
- outBuffer.put(data, offset, length);
- }
-
-
- /**
- * Flushes this output stream and forces any buffered output bytes to be
- * written out. If any data remains in the payload it is committed to the
- * service. Data is queued for writing and forced out to the service
- * before the call returns.
- */
- @Override
- public void flush() throws IOException {
-
- if (closed) {
- // calling close() after the stream is closed starts with call to flush()
- return;
- }
-
- addBlockUploadCommand();
-
- if (committedBlobLength.get() < blobLength) {
- try {
- // wait until the block list is committed
- addFlushCommand().await();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete.
- */
- @Override
- public void hsync() throws IOException {
- // when block compaction is disabled, hsync is empty function
- if (compactionEnabled) {
- flush();
- }
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete.
- */
- @Override
- public void hflush() throws IOException {
- // when block compaction is disabled, hflush is empty function
- if (compactionEnabled) {
- flush();
- }
- }
-
- /**
- * The Synchronization capabilities of this stream depend upon the compaction
- * policy.
- * @param capability string to query the stream support for.
- * @return true for hsync and hflush when compaction is enabled.
- */
- @Override
- public boolean hasCapability(String capability) {
- if (!compactionEnabled) {
- return false;
- }
- return StoreImplementationUtils.isProbeForSyncable(capability);
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete. Close the access to the stream and
- * shutdown the upload thread pool.
- * If the blob was created, its lease will be released.
- * Any error encountered caught in threads and stored will be rethrown here
- * after cleanup.
- */
- @Override
- public synchronized void close() throws IOException {
-
- LOG.debug("close {} ", key);
-
- if (closed) {
- return;
- }
-
- // Upload the last block regardless of compactionEnabled flag
- flush();
-
- // Initiates an orderly shutdown in which previously submitted tasks are
- // executed.
- ioThreadPool.shutdown();
-
- try {
- // wait up to CLOSE_UPLOAD_DELAY minutes to upload all the blocks
- if (!ioThreadPool.awaitTermination(CLOSE_UPLOAD_DELAY, TimeUnit.MINUTES)) {
- LOG.error("Time out occurred while close() is waiting for IO request to"
- + " finish in append"
- + " for blob : {}",
- key);
- NativeAzureFileSystemHelper.logAllLiveStackTraces();
- throw new AzureException("Timed out waiting for IO requests to finish");
- }
- } catch(InterruptedException ex) {
- Thread.currentThread().interrupt();
- }
-
- // release the lease
- if (firstError.get() == null && blobExist) {
- try {
- lease.free();
- } catch (StorageException ex) {
- LOG.debug("Lease free update blob {} encountered Storage Exception:"
- + " {} Error Code : {}",
- key,
- ex,
- ex.getErrorCode());
- maybeSetFirstError(new AzureException(ex));
- }
- }
-
- closed = true;
-
- // finally, throw the first exception raised if it has not
- // been thrown elsewhere.
- if (firstError.get() != null && !firstErrorThrown) {
- throw firstError.get();
- }
- }
-
- /**
- * Helper method used to generate the blockIDs. The algorithm used is similar
- * to the Azure storage SDK.
- */
- private void setBlocksCountAndBlockIdPrefix(List<BlockEntry> blockEntries) {
-
- if (nextBlockCount == UNSET_BLOCKS_COUNT && blockIdPrefix == null) {
-
- Random sequenceGenerator = new Random();
-
- String blockZeroBlockId = (!blockEntries.isEmpty())
- ? blockEntries.get(0).getId()
- : "";
- String prefix = UUID.randomUUID().toString() + "-";
- String sampleNewerVersionBlockId = generateNewerVersionBlockId(prefix,
- 0);
-
- if (!blockEntries.isEmpty()
- && blockZeroBlockId.length() < sampleNewerVersionBlockId.length()) {
-
- // If blob has already been created with 2.2.0, append subsequent blocks
- // with older version (2.2.0) blockId compute nextBlockCount, the way it
- // was done before; and don't use blockIdPrefix
- this.blockIdPrefix = "";
- nextBlockCount = (long) (sequenceGenerator.nextInt(Integer.MAX_VALUE))
- + sequenceGenerator.nextInt(
- Integer.MAX_VALUE - MAX_BLOCK_COUNT);
- nextBlockCount += blockEntries.size();
-
- } else {
-
- // If there are no existing blocks, create the first block with newer
- // version (4.2.0) blockId. If blob has already been created with 4.2.0,
- // append subsequent blocks with newer version (4.2.0) blockId
- this.blockIdPrefix = prefix;
- nextBlockCount = blockEntries.size();
- }
- }
- }
-
- /**
- * Helper method that generates the next block id for uploading a block to
- * azure storage.
- * @return String representing the block ID generated.
- * @throws IOException if the stream is in invalid state
- */
- private String generateBlockId() throws IOException {
-
- if (nextBlockCount == UNSET_BLOCKS_COUNT || blockIdPrefix == null) {
- throw new AzureException(
- "Append Stream in invalid state. nextBlockCount not set correctly");
- }
-
- return (!blockIdPrefix.isEmpty())
- ? generateNewerVersionBlockId(blockIdPrefix, nextBlockCount++)
- : generateOlderVersionBlockId(nextBlockCount++);
- }
-
- /**
- * Helper method that generates an older (2.2.0) version blockId.
- * @return String representing the block ID generated.
- */
- private String generateOlderVersionBlockId(long id) {
-
- byte[] blockIdInBytes = new byte[8];
- for (int m = 0; m < 8; m++) {
- blockIdInBytes[7 - m] = (byte) ((id >> (8 * m)) & 0xFF);
- }
-
- return new String(
- Base64.encodeBase64(blockIdInBytes),
- StandardCharsets.UTF_8);
- }
-
- /**
- * Helper method that generates an newer (4.2.0) version blockId.
- * @return String representing the block ID generated.
- */
- private String generateNewerVersionBlockId(String prefix, long id) {
-
- String blockIdSuffix = String.format("%06d", id);
- byte[] blockIdInBytes =
- (prefix + blockIdSuffix).getBytes(StandardCharsets.UTF_8);
- return new String(Base64.encodeBase64(blockIdInBytes), StandardCharsets.UTF_8);
- }
-
- /**
- * This is shared between upload block Runnable and CommitBlockList. The
- * method captures retry logic
- * @param blockId block name
- * @param dataPayload block content
- */
- private void writeBlockRequestInternal(String blockId,
- ByteBuffer dataPayload,
- boolean bufferPoolBuffer) {
- IOException lastLocalException = null;
-
- int uploadRetryAttempts = 0;
- while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
- try {
- long startTime = System.nanoTime();
-
- blob.uploadBlock(blockId, accessCondition, new ByteArrayInputStream(
- dataPayload.array()), dataPayload.position(),
- new BlobRequestOptions(), opContext);
-
- LOG.debug("upload block finished for {} ms. block {} ",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startTime), blockId);
- break;
-
- } catch(Exception ioe) {
- LOG.debug("Encountered exception during uploading block for Blob {}"
- + " Exception : {}", key, ioe);
- uploadRetryAttempts++;
- lastLocalException = new AzureException(
- "Encountered Exception while uploading block: " + ioe, ioe);
- try {
- Thread.sleep(
- BLOCK_UPLOAD_RETRY_INTERVAL * (uploadRetryAttempts + 1));
- } catch(InterruptedException ie) {
- Thread.currentThread().interrupt();
- break;
- }
- }
- }
-
- if (bufferPoolBuffer) {
- poolReadyByteBuffers.putBuffer(dataPayload);
- }
-
- if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
- maybeSetFirstError(lastLocalException);
- }
- }
-
- /**
- * Set {@link #firstError} to the exception if it is not already set.
- * @param exception exception to save
- */
- private void maybeSetFirstError(IOException exception) {
- firstError.compareAndSet(null, exception);
- }
-
-
- /**
- * Throw the first error caught if it has not been raised already
- * @throws IOException if one is caught and needs to be thrown.
- */
- private void maybeThrowFirstError() throws IOException {
- if (firstError.get() != null) {
- firstErrorThrown = true;
- throw firstError.get();
- }
- }
-
- /**
- * Write block list. The method captures retry logic
- */
- private void writeBlockListRequestInternal() {
-
- IOException lastLocalException = null;
-
- int uploadRetryAttempts = 0;
- while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
- try {
-
- long startTime = System.nanoTime();
-
- blob.commitBlockList(blockEntries, accessCondition,
- new BlobRequestOptions(), opContext);
-
- LOG.debug("Upload block list took {} ms for blob {} ",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startTime), key);
- break;
-
- } catch(Exception ioe) {
- LOG.debug("Encountered exception during uploading block for Blob {}"
- + " Exception : {}", key, ioe);
- uploadRetryAttempts++;
- lastLocalException = new AzureException(
- "Encountered Exception while uploading block: " + ioe, ioe);
- try {
- Thread.sleep(
- BLOCK_UPLOAD_RETRY_INTERVAL * (uploadRetryAttempts + 1));
- } catch(InterruptedException ie) {
- Thread.currentThread().interrupt();
- break;
- }
- }
- }
-
- if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
- maybeSetFirstError(lastLocalException);
- }
- }
-
- /**
- * A ThreadFactory that creates uploader thread with
- * meaningful names helpful for debugging purposes.
- */
- class UploaderThreadFactory implements ThreadFactory {
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new SubjectInheritingThread(r);
- t.setName(String.format("%s-%d", THREAD_ID_PREFIX,
- threadSequenceNumber.getAndIncrement()));
- return t;
- }
- }
-
- /**
- * Upload block commands.
- */
- private class UploadBlockCommand extends UploadCommand {
-
- // the block content for upload
- private final ByteBuffer payload;
-
- // description of the block
- private final BlockEntry entry;
-
- UploadBlockCommand(String blockId, ByteBuffer payload) {
-
- super(blobLength);
-
- BlockEntry blockEntry = new BlockEntry(blockId);
- blockEntry.setSize(payload.position());
- blockEntry.setSearchMode(BlockSearchMode.LATEST);
-
- this.payload = payload;
- this.entry = blockEntry;
-
- uncommittedBlockEntries.add(blockEntry);
- }
-
- /**
- * Execute command.
- */
- void execute() throws InterruptedException {
-
- uploadingSemaphore.acquire(1);
- writeBlockRequestInternal(entry.getId(), payload, true);
- uploadingSemaphore.release(1);
-
- }
-
- void dump() {
- LOG.debug("upload block {} size: {} for blob {}",
- entry.getId(),
- entry.getSize(),
- key);
- }
- }
-
- /**
- * Upload blob block list commands.
- */
- private class UploadBlockListCommand extends UploadCommand {
-
- private BlockEntry lastBlock = null;
-
- UploadBlockListCommand() {
- super(blobLength);
-
- if (!uncommittedBlockEntries.isEmpty()) {
- lastBlock = uncommittedBlockEntries.getLast();
- }
- }
-
- void awaitAsDependent() throws InterruptedException {
- // empty. later commit block does not need to wait previous commit block
- // lists.
- }
-
- void dump() {
- LOG.debug("commit block list with {} blocks for blob {}",
- uncommittedBlockEntries.size(), key);
- }
-
- /**
- * Execute command.
- */
- public void execute() throws InterruptedException, IOException {
-
- if (committedBlobLength.get() >= getCommandBlobOffset()) {
- LOG.debug("commit already applied for {}", key);
- return;
- }
-
- if (lastBlock == null) {
- LOG.debug("nothing to commit for {}", key);
- return;
- }
-
- LOG.debug("active commands: {} for {}", activeBlockCommands.size(), key);
-
- for (UploadCommand activeCommand : activeBlockCommands) {
- if (activeCommand.getCommandBlobOffset() < getCommandBlobOffset()) {
- activeCommand.dump();
- activeCommand.awaitAsDependent();
- } else {
- break;
- }
- }
-
- // stop all uploads until the block list is committed
- uploadingSemaphore.acquire(MAX_NUMBER_THREADS_IN_THREAD_POOL);
-
- BlockEntry uncommittedBlock;
- do {
- uncommittedBlock = uncommittedBlockEntries.poll();
- blockEntries.add(uncommittedBlock);
- } while (uncommittedBlock != lastBlock);
-
- if (blockEntries.size() > activateCompactionBlockCount) {
- LOG.debug("Block compaction: activated with {} blocks for {}",
- blockEntries.size(), key);
-
- // Block compaction
- long startCompaction = System.nanoTime();
- blockCompaction();
- LOG.debug("Block compaction finished for {} ms with {} blocks for {}",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startCompaction),
- blockEntries.size(), key);
- }
-
- writeBlockListRequestInternal();
-
- uploadingSemaphore.release(MAX_NUMBER_THREADS_IN_THREAD_POOL);
-
- // remove blocks previous commands
- for (Iterator<UploadCommand> it = activeBlockCommands.iterator();
- it.hasNext();) {
- UploadCommand activeCommand = it.next();
- if (activeCommand.getCommandBlobOffset() <= getCommandBlobOffset()) {
- it.remove();
- } else {
- break;
- }
- }
-
- committedBlobLength.set(getCommandBlobOffset());
- }
-
- /**
- * Internal output stream with read access to the internal buffer.
- */
- private class ByteArrayOutputStreamInternal extends ByteArrayOutputStream {
-
- ByteArrayOutputStreamInternal(int size) {
- super(size);
- }
-
- byte[] getByteArray() {
- return buf;
- }
- }
-
- /**
- * Block compaction process.
- *
- * Block compaction is only enabled when the number of blocks exceeds
- * activateCompactionBlockCount. The algorithm searches for the longest
- * segment [b..e) where (e-b) > 2 && |b| + |b+1| ... |e-1| < maxBlockSize
- * such that size(b1) + size(b2) + ... + size(bn) < maximum-block-size.
- * It then downloads the blocks in the sequence, concatenates the data to
- * form a single block, uploads this new block, and updates the block
- * list to replace the sequence of blocks with the new block.
- */
- private void blockCompaction() throws IOException {
- //current segment [segmentBegin, segmentEnd) and file offset/size of the
- // current segment
- int segmentBegin = 0, segmentEnd = 0;
- long segmentOffsetBegin = 0, segmentOffsetEnd = 0;
-
- //longest segment [maxSegmentBegin, maxSegmentEnd) and file offset/size of
- // the longest segment
- int maxSegmentBegin = 0, maxSegmentEnd = 0;
- long maxSegmentOffsetBegin = 0, maxSegmentOffsetEnd = 0;
-
- for (BlockEntry block : blockEntries) {
- segmentEnd++;
- segmentOffsetEnd += block.getSize();
- if (segmentOffsetEnd - segmentOffsetBegin > maxBlockSize.get()) {
- if (segmentEnd - segmentBegin > 2) {
- if (maxSegmentEnd - maxSegmentBegin < segmentEnd - segmentBegin) {
- maxSegmentBegin = segmentBegin;
- maxSegmentEnd = segmentEnd;
- maxSegmentOffsetBegin = segmentOffsetBegin;
- maxSegmentOffsetEnd = segmentOffsetEnd - block.getSize();
- }
- }
- segmentBegin = segmentEnd - 1;
- segmentOffsetBegin = segmentOffsetEnd - block.getSize();
- }
- }
-
- if (maxSegmentEnd - maxSegmentBegin > 1) {
-
- LOG.debug("Block compaction: {} blocks for {}",
- maxSegmentEnd - maxSegmentBegin, key);
-
- // download synchronously all the blocks from the azure storage
- ByteArrayOutputStreamInternal blockOutputStream
- = new ByteArrayOutputStreamInternal(maxBlockSize.get());
-
- try {
- long length = maxSegmentOffsetEnd - maxSegmentOffsetBegin;
- blob.downloadRange(maxSegmentOffsetBegin, length, blockOutputStream,
- new BlobRequestOptions(), opContext);
- } catch(StorageException ex) {
- LOG.error(
- "Storage exception encountered during block compaction phase"
- + " : {} Storage Exception : {} Error Code: {}",
- key, ex, ex.getErrorCode());
- throw new AzureException(
- "Encountered Exception while committing append blocks " + ex, ex);
- }
-
- // upload synchronously new block to the azure storage
- String blockId = generateBlockId();
-
- ByteBuffer byteBuffer = ByteBuffer.wrap(
- blockOutputStream.getByteArray());
- byteBuffer.position(blockOutputStream.size());
-
- writeBlockRequestInternal(blockId, byteBuffer, false);
-
- // replace blocks from the longest segment with new block id
- blockEntries.subList(maxSegmentBegin + 1, maxSegmentEnd - 1).clear();
- BlockEntry newBlock = blockEntries.get(maxSegmentBegin);
- newBlock.setId(blockId);
- newBlock.setSearchMode(BlockSearchMode.LATEST);
- newBlock.setSize(maxSegmentOffsetEnd - maxSegmentOffsetBegin);
- }
- }
- }
-
- /**
- * Prepare block upload command and queue the command in thread pool executor.
- */
- private synchronized void addBlockUploadCommand() throws IOException {
-
- maybeThrowFirstError();
-
- if (blobExist && lease.isFreed()) {
- throw new AzureException(String.format(
- "Attempting to upload a block on blob : %s "
- + " that does not have lease on the Blob. Failing upload", key));
- }
-
- int blockSize = outBuffer.position();
- if (blockSize > 0) {
- UploadCommand command = new UploadBlockCommand(generateBlockId(),
- outBuffer);
- activeBlockCommands.add(command);
-
- blobLength += blockSize;
- outBuffer = poolReadyByteBuffers.getBuffer(false, maxBlockSize.get());
-
- ioThreadPool.execute(new WriteRequest(command));
-
- }
- }
-
- /**
- * Prepare block list commit command and queue the command in thread pool
- * executor.
- */
- private synchronized UploadCommand addFlushCommand() throws IOException {
-
- maybeThrowFirstError();
-
- if (blobExist && lease.isFreed()) {
- throw new AzureException(
- String.format("Attempting to upload block list on blob : %s"
- + " that does not have lease on the Blob. Failing upload", key));
- }
-
- UploadCommand command = new UploadBlockListCommand();
- activeBlockCommands.add(command);
-
- ioThreadPool.execute(new WriteRequest(command));
-
- return command;
- }
-
- /**
- * Runnable instance that uploads the block of data to azure storage.
- */
- private class WriteRequest implements Runnable {
- private final UploadCommand command;
-
- WriteRequest(UploadCommand command) {
- this.command = command;
- }
-
- @Override
- public void run() {
-
- try {
- command.dump();
- long startTime = System.nanoTime();
- command.execute();
- command.setCompleted();
- LOG.debug("command finished for {} ms",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startTime));
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- LOG.debug(
- "Encountered exception during execution of command for Blob :"
- + " {} Exception : {}", key, ex);
- firstError.compareAndSet(null, new AzureException(ex));
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java
deleted file mode 100644
index 02ee8d6eac6..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-
-/**
- * Encapsulates the BlobInputStream used by block blobs and adds support for
- * random access and seek. Random access performance is improved by several
- * orders of magnitude.
- */
-final class BlockBlobInputStream extends FSInputStream {
- private final CloudBlockBlobWrapper blob;
- private final BlobRequestOptions options;
- private final OperationContext opContext;
- private final boolean bufferedPreadDisabled;
- private InputStream blobInputStream = null;
- private int minimumReadSizeInBytes = 0;
- private long streamPositionAfterLastRead = -1;
- // position of next network read within stream
- private long streamPosition = 0;
- // length of stream
- private long streamLength = 0;
- private boolean closed = false;
- // internal buffer, re-used for performance optimization
- private byte[] streamBuffer;
- // zero-based offset within streamBuffer of current read position
- private int streamBufferPosition;
- // length of data written to streamBuffer, streamBuffer may be larger
- private int streamBufferLength;
-
- /**
- * Creates a seek-able stream for reading from block blobs.
- * @param blob a block blob reference.
- * @param options the blob request options.
- * @param opContext the blob operation context.
- * @throws IOException IO failure
- */
- BlockBlobInputStream(CloudBlockBlobWrapper blob, BlobRequestOptions options,
- OperationContext opContext, boolean bufferedPreadDisabled)
- throws IOException {
- this.blob = blob;
- this.options = options;
- this.opContext = opContext;
- this.bufferedPreadDisabled = bufferedPreadDisabled;
-
- this.minimumReadSizeInBytes = blob.getStreamMinimumReadSizeInBytes();
-
- try {
- this.blobInputStream = blob.openInputStream(options, opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
-
- this.streamLength = blob.getProperties().getLength();
- }
-
- private void checkState() throws IOException {
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
- }
-
- /**
- * Reset the internal stream buffer but do not release the memory.
- * The buffer can be reused to avoid frequent memory allocations of
- * a large buffer.
- */
- private void resetStreamBuffer() {
- streamBufferPosition = 0;
- streamBufferLength = 0;
- }
-
- /**
- * Gets the read position of the stream.
- * @return the zero-based byte offset of the read position.
- * @throws IOException IO failure
- */
- @Override
- public synchronized long getPos() throws IOException {
- checkState();
- return (streamBuffer != null)
- ? streamPosition - streamBufferLength + streamBufferPosition
- : streamPosition;
- }
-
- /**
- * Sets the read position of the stream.
- * @param pos a zero-based byte offset in the stream.
- * @throws EOFException if read is out of range
- */
- @Override
- public synchronized void seek(long pos) throws IOException {
- checkState();
- if (pos < 0) {
- throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK + " " + pos);
- }
- if (pos > streamLength) {
- throw new EOFException(
- FSExceptionMessages.CANNOT_SEEK_PAST_EOF + " " + pos);
- }
-
- // calculate offset between the target and current position in the stream
- long offset = pos - getPos();
-
- if (offset == 0) {
- // no=op, no state change
- return;
- }
-
- if (offset > 0) {
- // forward seek, data can be skipped as an optimization
- if (skip(offset) != offset) {
- throw new EOFException(FSExceptionMessages.EOF_IN_READ_FULLY);
- }
- return;
- }
-
- // reverse seek, offset is negative
- if (streamBuffer != null) {
- if (streamBufferPosition + offset >= 0) {
- // target position is inside the stream buffer,
- // only need to move backwards within the stream buffer
- streamBufferPosition += offset;
- } else {
- // target position is outside the stream buffer,
- // need to reset stream buffer and move position for next network read
- resetStreamBuffer();
- streamPosition = pos;
- }
- } else {
- streamPosition = pos;
- }
-
- // close BlobInputStream after seek is invoked because BlobInputStream
- // does not support seek
- closeBlobInputStream();
- }
-
- /**
- * Seeks an secondary copy of the data. This method is not supported.
- * @param targetPos a zero-based byte offset in the stream.
- * @return false
- * @throws IOException IO failure
- */
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- /**
- * Gets the number of bytes that can be read (or skipped over) without
- * performing a network operation.
- * @throws IOException IO failure
- */
- @Override
- public synchronized int available() throws IOException {
- checkState();
- if (blobInputStream != null) {
- return blobInputStream.available();
- } else {
- return (streamBuffer == null)
- ? 0
- : streamBufferLength - streamBufferPosition;
- }
- }
-
- private void closeBlobInputStream() throws IOException {
- if (blobInputStream != null) {
- try {
- blobInputStream.close();
- } finally {
- blobInputStream = null;
- }
- }
- }
-
- /**
- * Closes this stream and releases any system resources associated with it.
- * @throws IOException IO failure
- */
- @Override
- public synchronized void close() throws IOException {
- closed = true;
- closeBlobInputStream();
- streamBuffer = null;
- streamBufferPosition = 0;
- streamBufferLength = 0;
- }
-
- private int doNetworkRead(byte[] buffer, int offset, int len)
- throws IOException {
- MemoryOutputStream outputStream;
- boolean needToCopy = false;
-
- if (streamPositionAfterLastRead == streamPosition) {
- // caller is reading sequentially, so initialize the stream buffer
- if (streamBuffer == null) {
- streamBuffer = new byte[(int) Math.min(minimumReadSizeInBytes,
- streamLength)];
- }
- resetStreamBuffer();
- outputStream = new MemoryOutputStream(streamBuffer, streamBufferPosition,
- streamBuffer.length);
- needToCopy = true;
- } else {
- outputStream = new MemoryOutputStream(buffer, offset, len);
- }
-
- long bytesToRead = Math.min(
- minimumReadSizeInBytes,
- Math.min(
- outputStream.capacity(),
- streamLength - streamPosition));
-
- try {
- blob.downloadRange(streamPosition, bytesToRead, outputStream, options,
- opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
-
- int bytesRead = outputStream.size();
- if (bytesRead > 0) {
- streamPosition += bytesRead;
- streamPositionAfterLastRead = streamPosition;
- int count = Math.min(bytesRead, len);
- if (needToCopy) {
- streamBufferLength = bytesRead;
- System.arraycopy(streamBuffer, streamBufferPosition, buffer, offset,
- count);
- streamBufferPosition += count;
- }
- return count;
- } else {
- // This may happen if the blob was modified after the length was obtained.
- throw new EOFException("End of stream reached unexpectedly.");
- }
- }
-
- @Override
- public int read(long position, byte[] buffer, int offset, int length)
- throws IOException {
- synchronized (this) {
- checkState();
- }
- if (!bufferedPreadDisabled) {
- // This will do a seek + read in which the streamBuffer will get used.
- return super.read(position, buffer, offset, length);
- }
- validatePositionedReadArgs(position, buffer, offset, length);
- if (length == 0) {
- return 0;
- }
- if (position >= streamLength) {
- throw new EOFException("position is beyond stream capacity");
- }
- MemoryOutputStream os = new MemoryOutputStream(buffer, offset, length);
- long bytesToRead = Math.min(minimumReadSizeInBytes,
- Math.min(os.capacity(), streamLength - position));
- try {
- blob.downloadRange(position, bytesToRead, os, options, opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
- int bytesRead = os.size();
- if (bytesRead == 0) {
- // This may happen if the blob was modified after the length was obtained.
- throw new EOFException("End of stream reached unexpectedly.");
- }
- return bytesRead;
- }
-
- /**
- * Reads up to <code>len</code> bytes of data from the input stream into an
- * array of bytes.
- * @param b a buffer into which the data is written.
- * @param offset a start offset into {@code buffer} where the data is written.
- * @param len the maximum number of bytes to be read.
- * @return the number of bytes written into {@code buffer}, or -1.
- * @throws IOException IO failure
- */
- @Override
- public synchronized int read(byte[] b, int offset, int len)
- throws IOException {
- checkState();
- NativeAzureFileSystemHelper.validateReadArgs(b, offset, len);
- if (blobInputStream != null) {
- int numberOfBytesRead = blobInputStream.read(b, offset, len);
- streamPosition += numberOfBytesRead;
- return numberOfBytesRead;
- } else {
- if (offset < 0 || len < 0 || len > b.length - offset) {
- throw new IndexOutOfBoundsException("read arguments out of range");
- }
- if (len == 0) {
- return 0;
- }
-
- int bytesRead = 0;
- int available = available();
- if (available > 0) {
- bytesRead = Math.min(available, len);
- System.arraycopy(streamBuffer, streamBufferPosition, b, offset,
- bytesRead);
- streamBufferPosition += bytesRead;
- }
-
- if (len == bytesRead) {
- return len;
- }
- if (streamPosition >= streamLength) {
- return (bytesRead > 0) ? bytesRead : -1;
- }
-
- offset += bytesRead;
- len -= bytesRead;
-
- return bytesRead + doNetworkRead(b, offset, len);
- }
- }
-
- /**
- * Reads the next byte of data from the stream.
- * @return the next byte of data, or -1
- * @throws IOException IO failure
- */
- @Override
- public int read() throws IOException {
- byte[] buffer = new byte[1];
- int numberOfBytesRead = read(buffer, 0, 1);
- return (numberOfBytesRead < 1) ? -1 : buffer[0];
- }
-
- /**
- * Skips over and discards n bytes of data from this input stream.
- * @param n the number of bytes to be skipped.
- * @return the actual number of bytes skipped.
- * @throws IOException IO failure
- * @throws IndexOutOfBoundsException if n is negative or if the sum of n
- * and the current value of getPos() is greater than the length of the stream.
- */
- @Override
- public synchronized long skip(long n) throws IOException {
- checkState();
-
- if (blobInputStream != null) {
- // blobInput stream is open; delegate the work to it
- long skipped = blobInputStream.skip(n);
- // update position to the actual skip value
- streamPosition += skipped;
- return skipped;
- }
-
- // no blob stream; implement the skip logic directly
- if (n < 0 || n > streamLength - getPos()) {
- throw new IndexOutOfBoundsException("skip range");
- }
-
- if (streamBuffer != null) {
- // there's a buffer, so seek with it
- if (n < streamBufferLength - streamBufferPosition) {
- // new range is in the buffer, so just update the buffer position
- // skip within the buffer.
- streamBufferPosition += (int) n;
- } else {
- // skip is out of range, so move position to ne value and reset
- // the buffer ready for the next read()
- streamPosition = getPos() + n;
- resetStreamBuffer();
- }
- } else {
- // no stream buffer; increment the stream position ready for
- // the next triggered connection & read
- streamPosition += n;
- }
- return n;
- }
-
- /**
- * An <code>OutputStream</code> backed by a user-supplied buffer.
- */
- static class MemoryOutputStream extends OutputStream {
- private final byte[] buffer;
- private final int offset;
- private final int length;
- private int writePosition;
-
- /**
- * Creates a <code>MemoryOutputStream</code> from a user-supplied buffer.
- * @param buffer an array of bytes.
- * @param offset a starting offset in <code>buffer</code> where the data
- * will be written.
- * @param length the maximum number of bytes to be written to the stream.
- */
- MemoryOutputStream(byte[] buffer, int offset, int length) {
- if (buffer == null) {
- throw new NullPointerException("buffer");
- }
- if (offset < 0 || length < 0 || length > buffer.length - offset) {
- throw new IndexOutOfBoundsException("offset out of range of buffer");
- }
- this.buffer = buffer;
- this.offset = offset;
- this.length = length;
- this.writePosition = offset;
- }
-
- /**
- * Gets the current size of the stream.
- */
- public synchronized int size() {
- return writePosition - offset;
- }
-
- /**
- * Gets the current capacity of the stream.
- */
- public synchronized int capacity() {
- return length;
- }
-
- /**
- * Writes the next byte to the stream.
- * @param b the byte to be written.
- * @throws IOException IO failure
- */
- public synchronized void write(int b) throws IOException {
- if (size() > length - 1) {
- throw new IOException("No space for more writes");
- }
- buffer[writePosition++] = (byte) b;
- }
-
- /**
- * Writes a range of bytes to the stream.
- * @param b a byte array.
- * @param off the start offset in <code>buffer</code> from which the data
- * is read.
- * @param length the number of bytes to be written.
- * @throws IOException IO failure
- */
- public synchronized void write(byte[] b, int off, int length)
- throws IOException {
- if (b == null) {
- throw new NullPointerException("Null buffer argument");
- }
- if (off < 0 || length < 0 || length > b.length - off) {
- throw new IndexOutOfBoundsException("array write offset");
- }
- System.arraycopy(b, off, buffer, writePosition, length);
- writePosition += length;
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java
deleted file mode 100644
index 5fa0ad8ae34..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
-
-/**
- * Class that provides caching for Authorize and getSasUri calls
- * @param <K> - The cache key type
- * @param <V> - The cached value type
- */
-public class CachingAuthorizer<K, V> {
-
- public static final Logger LOG = LoggerFactory
- .getLogger(CachingAuthorizer.class);
-
- private Cache<K, V> cache;
- private boolean isEnabled = false;
- private long cacheEntryExpiryPeriodInMinutes;
- private String label;
-
- public static final String KEY_AUTH_SERVICE_CACHING_ENABLE =
- "fs.azure.authorization.caching.enable";
-
- public static final boolean KEY_AUTH_SERVICE_CACHING_ENABLE_DEFAULT = false;
-
- public static final String KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES =
- "fs.azure.authorization.caching.maxentries";
-
- public static final int KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES_DEFAULT = 512;
-
- public CachingAuthorizer(long ttlInMinutes, String label) {
- cacheEntryExpiryPeriodInMinutes = ttlInMinutes;
- this.label = label;
- if (cacheEntryExpiryPeriodInMinutes <= 0) {
- isEnabled = false;
- }
- }
-
-
- public void init(Configuration conf) {
-
- isEnabled = conf.getBoolean(KEY_AUTH_SERVICE_CACHING_ENABLE, KEY_AUTH_SERVICE_CACHING_ENABLE_DEFAULT);
-
- if (isEnabled) {
- LOG.debug("{} : Initializing CachingAuthorizer instance", label);
- cache = CacheBuilder.newBuilder()
- .maximumSize(
- conf.getInt(
- KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES,
- KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES_DEFAULT
- )
- )
- .expireAfterWrite(cacheEntryExpiryPeriodInMinutes, TimeUnit.MINUTES)
- .build();
- }
- }
-
- /**
- * @param key - Cache key
- * @return null on cache-miss. true/false on cache-hit
- */
- public V get(K key) {
- if (!isEnabled) {
- return null;
- }
-
- V result = cache.getIfPresent(key);
- if (result == null) {
- LOG.debug("{}: CACHE MISS: {}", label, key.toString());
- }
- else {
- LOG.debug("{}: CACHE HIT: {}, {}", label, key.toString(), result.toString());
- }
- return result;
- }
-
- public void put(K key, V value) {
- if (isEnabled) {
- LOG.debug("{}: CACHE PUT: {}, {}", label, key.toString(), value.toString());
- cache.put(key, value);
- }
- }
-
- public void clear() {
- if (isEnabled) {
- cache.invalidateAll();
- }
- }
-}
-
-/**
- * POJO representing the cache key for authorization calls
- */
-class CachedAuthorizerEntry {
-
- private String path;
- private String accessType;
- private String owner;
-
- CachedAuthorizerEntry(String path, String accessType, String owner) {
- this.path = path;
- this.accessType = accessType;
- this.owner = owner;
- }
-
- public String getPath() {
- return path;
- }
-
- public String getAccessType() {
- return accessType;
- }
-
- public String getOwner() {
- return owner;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
-
- if (o == null) {
- return false;
- }
-
- if (!(o instanceof CachedAuthorizerEntry)) {
- return false;
- }
-
- CachedAuthorizerEntry c = (CachedAuthorizerEntry) o;
- return
- this.getPath().equals(c.getPath())
- && this.getAccessType().equals(c.getAccessType())
- && this.getOwner().equals(c.getOwner());
- }
-
- @Override
- public int hashCode() {
- return this.toString().hashCode();
- }
-
- @Override
- public String toString() {
- return path + ":" + accessType + ":" + owner;
- }
-
-}
-
-
-/**
- * POJO representing the cache key for sas-key calls
- */
-class CachedSASKeyEntry {
-
- private String storageAccount;
- private String container;
- private String path;
-
- CachedSASKeyEntry(String storageAccount, String container, String path) {
- this.storageAccount = storageAccount;
- this.container = container;
- this.path = path;
- }
-
- public String getStorageAccount() {
- return storageAccount;
- }
-
- public String getContainer() {
- return container;
- }
-
- public String getPath() {
- return path;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
-
- if (o == null) {
- return false;
- }
-
- if (!(o instanceof CachedSASKeyEntry)) {
- return false;
- }
-
- CachedSASKeyEntry c = (CachedSASKeyEntry) o;
- return
- this.getStorageAccount().equals(c.getStorageAccount())
- && this.getContainer().equals(c.getContainer())
- && this.getPath().equals(c.getPath());
- }
-
- @Override
- public int hashCode() {
- return this.toString().hashCode();
- }
-
- @Override
- public String toString() {
- return storageAccount + ":" + container + ":" + path;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
deleted file mode 100644
index 3c4803cd3df..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.util.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * Throttles storage operations to minimize errors and maximum throughput. This
- * improves throughput by as much as 35% when the service throttles requests due
- * to exceeding account level ingress or egress limits.
- */
-@InterfaceAudience.Private
-class ClientThrottlingAnalyzer {
- private static final Logger LOG = LoggerFactory.getLogger(
- ClientThrottlingAnalyzer.class);
- private static final int DEFAULT_ANALYSIS_PERIOD_MS = 10 * 1000;
- private static final int MIN_ANALYSIS_PERIOD_MS = 1000;
- private static final int MAX_ANALYSIS_PERIOD_MS = 30000;
- private static final double MIN_ACCEPTABLE_ERROR_PERCENTAGE = .1;
- private static final double MAX_EQUILIBRIUM_ERROR_PERCENTAGE = 1;
- private static final double RAPID_SLEEP_DECREASE_FACTOR = .75;
- private static final double RAPID_SLEEP_DECREASE_TRANSITION_PERIOD_MS = 150
- * 1000;
- private static final double SLEEP_DECREASE_FACTOR = .975;
- private static final double SLEEP_INCREASE_FACTOR = 1.05;
- private int analysisPeriodMs;
-
- private volatile int sleepDuration = 0;
- private long consecutiveNoErrorCount = 0;
- private String name = null;
- private Timer timer = null;
- private AtomicReference<BlobOperationMetrics> blobMetrics = null;
-
- private ClientThrottlingAnalyzer() {
- // hide default constructor
- }
-
- /**
- * Creates an instance of the <code>ClientThrottlingAnalyzer</code> class with
- * the specified name.
- *
- * @param name a name used to identify this instance.
- *
- * @throws IllegalArgumentException if name is null or empty.
- */
- ClientThrottlingAnalyzer(String name) throws IllegalArgumentException {
- this(name, DEFAULT_ANALYSIS_PERIOD_MS);
- }
-
- /**
- * Creates an instance of the <code>ClientThrottlingAnalyzer</code> class with
- * the specified name and period.
- *
- * @param name A name used to identify this instance.
- *
- * @param period The frequency, in milliseconds, at which metrics are
- * analyzed.
- *
- * @throws IllegalArgumentException
- * If name is null or empty.
- * If period is less than 1000 or greater than 30000 milliseconds.
- */
- ClientThrottlingAnalyzer(String name, int period)
- throws IllegalArgumentException {
- Preconditions.checkArgument(
- StringUtils.isNotEmpty(name),
- "The argument 'name' cannot be null or empty.");
- Preconditions.checkArgument(
- period >= MIN_ANALYSIS_PERIOD_MS && period <= MAX_ANALYSIS_PERIOD_MS,
- "The argument 'period' must be between 1000 and 30000.");
- this.name = name;
- this.analysisPeriodMs = period;
- this.blobMetrics = new AtomicReference<BlobOperationMetrics>(
- new BlobOperationMetrics(System.currentTimeMillis()));
- this.timer = new Timer(
- String.format("wasb-timer-client-throttling-analyzer-%s", name), true);
- this.timer.schedule(new TimerTaskImpl(),
- analysisPeriodMs,
- analysisPeriodMs);
- }
-
- /**
- * Updates metrics with results from the current storage operation.
- *
- * @param count The count of bytes transferred.
- *
- * @param isFailedOperation True if the operation failed; otherwise false.
- */
- public void addBytesTransferred(long count, boolean isFailedOperation) {
- BlobOperationMetrics metrics = blobMetrics.get();
- if (isFailedOperation) {
- metrics.bytesFailed.addAndGet(count);
- metrics.operationsFailed.incrementAndGet();
- } else {
- metrics.bytesSuccessful.addAndGet(count);
- metrics.operationsSuccessful.incrementAndGet();
- }
- }
-
- /**
- * Suspends the current storage operation, as necessary, to reduce throughput.
- */
- public void suspendIfNecessary() {
- int duration = sleepDuration;
- if (duration > 0) {
- try {
- Thread.sleep(duration);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- @VisibleForTesting
- int getSleepDuration() {
- return sleepDuration;
- }
-
- private int analyzeMetricsAndUpdateSleepDuration(BlobOperationMetrics metrics,
- int sleepDuration) {
- final double percentageConversionFactor = 100;
- double bytesFailed = metrics.bytesFailed.get();
- double bytesSuccessful = metrics.bytesSuccessful.get();
- double operationsFailed = metrics.operationsFailed.get();
- double operationsSuccessful = metrics.operationsSuccessful.get();
- double errorPercentage = (bytesFailed <= 0)
- ? 0
- : percentageConversionFactor
- * bytesFailed
- / (bytesFailed + bytesSuccessful);
- long periodMs = metrics.endTime - metrics.startTime;
-
- double newSleepDuration;
-
- if (errorPercentage < MIN_ACCEPTABLE_ERROR_PERCENTAGE) {
- ++consecutiveNoErrorCount;
- // Decrease sleepDuration in order to increase throughput.
- double reductionFactor =
- (consecutiveNoErrorCount * analysisPeriodMs
- >= RAPID_SLEEP_DECREASE_TRANSITION_PERIOD_MS)
- ? RAPID_SLEEP_DECREASE_FACTOR
- : SLEEP_DECREASE_FACTOR;
-
- newSleepDuration = sleepDuration * reductionFactor;
- } else if (errorPercentage < MAX_EQUILIBRIUM_ERROR_PERCENTAGE) {
- // Do not modify sleepDuration in order to stabilize throughput.
- newSleepDuration = sleepDuration;
- } else {
- // Increase sleepDuration in order to minimize error rate.
- consecutiveNoErrorCount = 0;
-
- // Increase sleep duration in order to reduce throughput and error rate.
- // First, calculate target throughput: bytesSuccessful / periodMs.
- // Next, calculate time required to send *all* data (assuming next period
- // is similar to previous) at the target throughput: (bytesSuccessful
- // + bytesFailed) * periodMs / bytesSuccessful. Next, subtract periodMs to
- // get the total additional delay needed.
- double additionalDelayNeeded = 5 * analysisPeriodMs;
- if (bytesSuccessful > 0) {
- additionalDelayNeeded = (bytesSuccessful + bytesFailed)
- * periodMs
- / bytesSuccessful
- - periodMs;
- }
-
- // amortize the additional delay needed across the estimated number of
- // requests during the next period
- newSleepDuration = additionalDelayNeeded
- / (operationsFailed + operationsSuccessful);
-
- final double maxSleepDuration = analysisPeriodMs;
- final double minSleepDuration = sleepDuration * SLEEP_INCREASE_FACTOR;
-
- // Add 1 ms to avoid rounding down and to decrease proximity to the server
- // side ingress/egress limit. Ensure that the new sleep duration is
- // larger than the current one to more quickly reduce the number of
- // errors. Don't allow the sleep duration to grow unbounded, after a
- // certain point throttling won't help, for example, if there are far too
- // many tasks/containers/nodes no amount of throttling will help.
- newSleepDuration = Math.max(newSleepDuration, minSleepDuration) + 1;
- newSleepDuration = Math.min(newSleepDuration, maxSleepDuration);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format(
- "%5.5s, %10d, %10d, %10d, %10d, %6.2f, %5d, %5d, %5d",
- name,
- (int) bytesFailed,
- (int) bytesSuccessful,
- (int) operationsFailed,
- (int) operationsSuccessful,
- errorPercentage,
- periodMs,
- (int) sleepDuration,
- (int) newSleepDuration));
- }
-
- return (int) newSleepDuration;
- }
-
- /**
- * Timer callback implementation for periodically analyzing metrics.
- */
- class TimerTaskImpl extends TimerTask {
- private AtomicInteger doingWork = new AtomicInteger(0);
-
- /**
- * Periodically analyzes a snapshot of the blob storage metrics and updates
- * the sleepDuration in order to appropriately throttle storage operations.
- */
- @Override
- public void run() {
- boolean doWork = false;
- try {
- doWork = doingWork.compareAndSet(0, 1);
-
- // prevent concurrent execution of this task
- if (!doWork) {
- return;
- }
-
- long now = System.currentTimeMillis();
- if (now - blobMetrics.get().startTime >= analysisPeriodMs) {
- BlobOperationMetrics oldMetrics = blobMetrics.getAndSet(
- new BlobOperationMetrics(now));
- oldMetrics.endTime = now;
- sleepDuration = analyzeMetricsAndUpdateSleepDuration(oldMetrics,
- sleepDuration);
- }
- }
- finally {
- if (doWork) {
- doingWork.set(0);
- }
- }
- }
- }
-
- /**
- * Stores blob operation metrics during each analysis period.
- */
- static class BlobOperationMetrics {
- private AtomicLong bytesFailed;
- private AtomicLong bytesSuccessful;
- private AtomicLong operationsFailed;
- private AtomicLong operationsSuccessful;
- private long endTime;
- private long startTime;
-
- BlobOperationMetrics(long startTime) {
- this.startTime = startTime;
- this.bytesFailed = new AtomicLong();
- this.bytesSuccessful = new AtomicLong();
- this.operationsFailed = new AtomicLong();
- this.operationsSuccessful = new AtomicLong();
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java
deleted file mode 100644
index 9da993bd237..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import com.microsoft.azure.storage.ErrorReceivingResponseEvent;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RequestResult;
-import com.microsoft.azure.storage.ResponseReceivedEvent;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.StorageEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.net.HttpURLConnection;
-
-/**
- * Throttles Azure Storage read and write operations to achieve maximum
- * throughput by minimizing errors. The errors occur when the account ingress
- * or egress limits are exceeded and the server-side throttles requests.
- * Server-side throttling causes the retry policy to be used, but the retry
- * policy sleeps for long periods of time causing the total ingress or egress
- * throughput to be as much as 35% lower than optimal. The retry policy is also
- * after the fact, in that it applies after a request fails. On the other hand,
- * the client-side throttling implemented here happens before requests are made
- * and sleeps just enough to minimize errors, allowing optimal ingress and/or
- * egress throughput.
- */
-@InterfaceAudience.Private
-final class ClientThrottlingIntercept {
- private static final Logger LOG = LoggerFactory.getLogger(
- ClientThrottlingIntercept.class);
- private static ClientThrottlingIntercept singleton = null;
- private ClientThrottlingAnalyzer readThrottler = null;
- private ClientThrottlingAnalyzer writeThrottler = null;
-
- // Hide default constructor
- private ClientThrottlingIntercept() {
- readThrottler = new ClientThrottlingAnalyzer("read");
- writeThrottler = new ClientThrottlingAnalyzer("write");
- LOG.debug("Client-side throttling is enabled for the WASB file system.");
- }
-
- static synchronized void initializeSingleton() {
- if (singleton == null) {
- singleton = new ClientThrottlingIntercept();
- }
- }
-
- static void hook(OperationContext context) {
- context.getErrorReceivingResponseEventHandler().addListener(
- new ErrorReceivingResponseEventHandler());
- context.getSendingRequestEventHandler().addListener(
- new SendingRequestEventHandler());
- context.getResponseReceivedEventHandler().addListener(
- new ResponseReceivedEventHandler());
- }
-
- private static void updateMetrics(HttpURLConnection conn,
- RequestResult result) {
- BlobOperationDescriptor.OperationType operationType
- = BlobOperationDescriptor.getOperationType(conn);
- int status = result.getStatusCode();
- long contentLength = 0;
- // If the socket is terminated prior to receiving a response, the HTTP
- // status may be 0 or -1. A status less than 200 or greater than or equal
- // to 500 is considered an error.
- boolean isFailedOperation = (status < HttpURLConnection.HTTP_OK
- || status >= java.net.HttpURLConnection.HTTP_INTERNAL_ERROR);
-
- switch (operationType) {
- case AppendBlock:
- case PutBlock:
- case PutPage:
- contentLength = BlobOperationDescriptor.getContentLengthIfKnown(conn,
- operationType);
- if (contentLength > 0) {
- singleton.writeThrottler.addBytesTransferred(contentLength,
- isFailedOperation);
- }
- break;
- case GetBlob:
- contentLength = BlobOperationDescriptor.getContentLengthIfKnown(conn,
- operationType);
- if (contentLength > 0) {
- singleton.readThrottler.addBytesTransferred(contentLength,
- isFailedOperation);
- }
- break;
- default:
- break;
- }
- }
-
- /**
- * Called when a network error occurs before the HTTP status and response
- * headers are received. Client-side throttling uses this to collect metrics.
- *
- * @param event The connection, operation, and request state.
- */
- public static void errorReceivingResponse(ErrorReceivingResponseEvent event) {
- updateMetrics((HttpURLConnection) event.getConnectionObject(),
- event.getRequestResult());
- }
-
- /**
- * Called before the Azure Storage SDK sends a request. Client-side throttling
- * uses this to suspend the request, if necessary, to minimize errors and
- * maximize throughput.
- *
- * @param event The connection, operation, and request state.
- */
- public static void sendingRequest(SendingRequestEvent event) {
- BlobOperationDescriptor.OperationType operationType
- = BlobOperationDescriptor.getOperationType(
- (HttpURLConnection) event.getConnectionObject());
- switch (operationType) {
- case GetBlob:
- singleton.readThrottler.suspendIfNecessary();
- break;
- case AppendBlock:
- case PutBlock:
- case PutPage:
- singleton.writeThrottler.suspendIfNecessary();
- break;
- default:
- break;
- }
- }
-
- /**
- * Called after the Azure Storage SDK receives a response. Client-side
- * throttling uses this to collect metrics.
- *
- * @param event The connection, operation, and request state.
- */
- public static void responseReceived(ResponseReceivedEvent event) {
- updateMetrics((HttpURLConnection) event.getConnectionObject(),
- event.getRequestResult());
- }
-
- /**
- * The ErrorReceivingResponseEvent is fired when the Azure Storage SDK
- * encounters a network error before the HTTP status and response headers are
- * received.
- */
- @InterfaceAudience.Private
- static class ErrorReceivingResponseEventHandler
- extends StorageEvent<ErrorReceivingResponseEvent> {
-
- /**
- * Called when a network error occurs before the HTTP status and response
- * headers are received. Client-side throttling uses this to collect
- * metrics.
- *
- * @param event The connection, operation, and request state.
- */
- @Override
- public void eventOccurred(ErrorReceivingResponseEvent event) {
- singleton.errorReceivingResponse(event);
- }
- }
-
- /**
- * The SendingRequestEvent is fired before the Azure Storage SDK sends a
- * request.
- */
- @InterfaceAudience.Private
- static class SendingRequestEventHandler
- extends StorageEvent<SendingRequestEvent> {
-
- /**
- * Called before the Azure Storage SDK sends a request. Client-side
- * throttling uses this to suspend the request, if necessary, to minimize
- * errors and maximize throughput.
- *
- * @param event The connection, operation, and request state.
- */
- @Override
- public void eventOccurred(SendingRequestEvent event) {
- singleton.sendingRequest(event);
- }
- }
-
- /**
- * The ResponseReceivedEvent is fired after the Azure Storage SDK receives a
- * response.
- */
- @InterfaceAudience.Private
- static class ResponseReceivedEventHandler
- extends StorageEvent<ResponseReceivedEvent> {
-
- /**
- * Called after the Azure Storage SDK receives a response. Client-side
- * throttling uses this
- * to collect metrics.
- *
- * @param event The connection, operation, and request state.
- */
- @Override
- public void eventOccurred(ResponseReceivedEvent event) {
- singleton.responseReceived(event);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
deleted file mode 100644
index cbf3ab96160..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-
-/**
- * <p>
- * Holds basic metadata for a file stored in a {@link NativeFileSystemStore}.
- * </p>
- */
-@InterfaceAudience.Private
-class FileMetadata extends FileStatus {
- // this is not final so that it can be cleared to save memory when not needed.
- private String key;
- private final BlobMaterialization blobMaterialization;
-
- /**
- * Constructs a FileMetadata object for a file.
- *
- * @param key
- * The key (path) to the file.
- * @param length
- * The length in bytes of the file.
- * @param lastModified
- * The last modified date (milliseconds since January 1, 1970 UTC.)
- * @param permissionStatus
- * The permission for the file.
- * @param blockSize
- * The Hadoop file block size.
- */
- public FileMetadata(String key, long length, long lastModified,
- PermissionStatus permissionStatus, final long blockSize) {
- super(length, false, 1, blockSize, lastModified, 0,
- permissionStatus.getPermission(),
- permissionStatus.getUserName(),
- permissionStatus.getGroupName(),
- null);
- this.key = key;
- // Files are never implicit.
- this.blobMaterialization = BlobMaterialization.Explicit;
- }
-
- /**
- * Constructs a FileMetadata object for a directory.
- *
- * @param key
- * The key (path) to the directory.
- * @param lastModified
- * The last modified date (milliseconds since January 1, 1970 UTC.)
- * @param permissionStatus
- * The permission for the directory.
- * @param blobMaterialization
- * Whether this is an implicit (no real blob backing it) or explicit
- * directory.
- * @param blockSize
- * The Hadoop file block size.
- */
- public FileMetadata(String key, long lastModified,
- PermissionStatus permissionStatus, BlobMaterialization blobMaterialization,
- final long blockSize) {
- super(0, true, 1, blockSize, lastModified, 0,
- permissionStatus.getPermission(),
- permissionStatus.getUserName(),
- permissionStatus.getGroupName(),
- null);
- this.key = key;
- this.blobMaterialization = blobMaterialization;
- }
-
- @Override
- public Path getPath() {
- Path p = super.getPath();
- if (p == null) {
- // Don't store this yet to reduce memory usage, as it will
- // stay in the Eden Space and later we will update it
- // with the full canonicalized path.
- p = NativeAzureFileSystem.keyToPath(key);
- }
- return p;
- }
-
- /**
- * Returns the Azure storage key for the file. Used internally by the framework.
- *
- * @return The key for the file.
- */
- public String getKey() {
- return key;
- }
-
- /**
- * Indicates whether this is an implicit directory (no real blob backing it)
- * or an explicit one.
- *
- * @return Implicit if this is an implicit directory, or Explicit if it's an
- * explicit directory or a file.
- */
- public BlobMaterialization getBlobMaterialization() {
- return blobMaterialization;
- }
-
- void removeKey() {
- key = null;
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java
deleted file mode 100644
index ed510f67cad..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * The interface that every Azure file system key provider must implement.
- */
-@InterfaceAudience.Private
-public interface KeyProvider {
- /**
- * Key providers must implement this method. Given a list of configuration
- * parameters for the specified Azure storage account, retrieve the plaintext
- * storage account key.
- *
- * @param accountName
- * the storage account name
- * @param conf
- * Hadoop configuration parameters
- * @return the plaintext storage account key
- * @throws KeyProviderException Thrown if there is a problem instantiating a
- * KeyProvider or retrieving a key using a KeyProvider object.
- */
- String getStorageAccountKey(String accountName, Configuration conf)
- throws KeyProviderException;
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java
deleted file mode 100644
index b65b2e6f42f..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Thrown if there is a problem instantiating a KeyProvider or retrieving a key
- * using a KeyProvider object.
- */
-@InterfaceAudience.Private
-public class KeyProviderException extends Exception {
- private static final long serialVersionUID = 1L;
-
- public KeyProviderException(String message) {
- super(message);
- }
-
- public KeyProviderException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public KeyProviderException(Throwable t) {
- super(t);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java
deleted file mode 100644
index b573457cd17..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.StringTokenizer;
-import java.util.TimeZone;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.SharedAccessAccountPermissions;
-import com.microsoft.azure.storage.SharedAccessAccountPolicy;
-import com.microsoft.azure.storage.SharedAccessAccountResourceType;
-import com.microsoft.azure.storage.SharedAccessAccountService;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/***
- * Local SAS Key Generation implementation. This class resides in
- * the same address space as the WASB driver.
- *
- * This class gets typically used for testing purposes.
- *
- */
-
-public class LocalSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
-
- public static final Logger LOG = LoggerFactory.getLogger(
- LocalSASKeyGeneratorImpl.class);
-
- /**
- * Map to cache CloudStorageAccount instances.
- */
- private Map<String, CloudStorageAccount> storageAccountMap;
- private CachingAuthorizer<CachedSASKeyEntry, URI> cache;
- private static final int HOURS_IN_DAY = 24;
-
- public LocalSASKeyGeneratorImpl(Configuration conf) {
- super(conf);
- storageAccountMap = new HashMap<String, CloudStorageAccount>();
- cache = new CachingAuthorizer<>(getSasKeyExpiryPeriod(), "SASKEY");
- cache.init(conf);
- }
-
- /**
- * Implementation to generate SAS Key for a container
- */
- @Override
- public URI getContainerSASUri(String accountName, String container)
- throws SASKeyGenerationException {
-
- LOG.debug("Retrieving Container SAS URI For {}@{}", container, accountName);
- try {
-
- CachedSASKeyEntry cacheKey = new CachedSASKeyEntry(accountName, container, "/");
- URI cacheResult = cache.get(cacheKey);
- if (cacheResult != null) {
- return cacheResult;
- }
-
- CloudStorageAccount account =
- getSASKeyBasedStorageAccountInstance(accountName);
- CloudBlobClient client = account.createCloudBlobClient();
- URI sasKey = client.getCredentials().transformUri(
- client.getContainerReference(container).getUri());
- cache.put(cacheKey, sasKey);
- return sasKey;
-
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while"
- + " generating SAS Key for container " + container + " inside "
- + "storage account " + accountName, stoEx);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException while"
- + " generating SAS Key for container " + container + " inside storage"
- + " account " + accountName, uriSyntaxEx);
- }
- }
-
- /**
- * Helper method that creates a CloudStorageAccount instance based on
- * SAS key for accountName
- *
- * @param accountName Storage Account Name
- * @return CloudStorageAccount instance created using SAS key for
- * the Storage Account.
- * @throws SASKeyGenerationException
- */
- private CloudStorageAccount getSASKeyBasedStorageAccountInstance(
- String accountName) throws SASKeyGenerationException {
- LOG.debug("Creating SAS key from account instance {}", accountName);
- try {
-
- String accountNameWithoutDomain =
- getAccountNameWithoutDomain(accountName);
-
- CloudStorageAccount account =
- getStorageAccountInstance(accountNameWithoutDomain,
- AzureNativeFileSystemStore.getAccountKeyFromConfiguration(
- accountName, getConf()));
-
- return new CloudStorageAccount(
- new StorageCredentialsSharedAccessSignature(
- account.generateSharedAccessSignature(
- getDefaultAccountAccessPolicy())), false,
- account.getEndpointSuffix(), accountNameWithoutDomain);
-
- } catch (KeyProviderException keyProviderEx) {
- throw new SASKeyGenerationException("Encountered KeyProviderException"
- + " while retrieving Storage key from configuration for account "
- + accountName, keyProviderEx);
- } catch (InvalidKeyException invalidKeyEx) {
- throw new SASKeyGenerationException("Encoutered InvalidKeyException "
- + "while generating Account level SAS key for account" + accountName,
- invalidKeyEx);
- } catch(StorageException storeEx) {
- throw new SASKeyGenerationException("Encoutered StorageException while "
- + "generating Account level SAS key for account" + accountName,
- storeEx);
- } catch(URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException for"
- + " account " + accountName, uriSyntaxEx);
- }
- }
-
- /**
- * Implementation for generation of Relative Path Blob SAS Uri.
- */
- @Override
- public URI getRelativeBlobSASUri(String accountName, String container,
- String relativePath) throws SASKeyGenerationException {
-
- CloudBlobContainer sc = null;
- CloudBlobClient client = null;
- CachedSASKeyEntry cacheKey = null;
-
- try {
-
- cacheKey = new CachedSASKeyEntry(accountName, container, relativePath);
- URI cacheResult = cache.get(cacheKey);
- if (cacheResult != null) {
- return cacheResult;
- }
-
- CloudStorageAccount account =
- getSASKeyBasedStorageAccountInstance(accountName);
- client = account.createCloudBlobClient();
- sc = client.getContainerReference(container);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException "
- + "while getting container references for container " + container
- + " inside storage account : " + accountName, uriSyntaxEx);
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while "
- + "getting container references for container " + container
- + " inside storage account : " + accountName, stoEx);
- }
-
- CloudBlockBlob blob = null;
- try {
- blob = sc.getBlockBlobReference(relativePath);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException while "
- + "getting Block Blob references for container " + container
- + " inside storage account : " + accountName, uriSyntaxEx);
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while "
- + "getting Block Blob references for container " + container
- + " inside storage account : " + accountName, stoEx);
- }
-
- try {
- URI sasKey = client.getCredentials().transformUri(blob.getUri());
- cache.put(cacheKey, sasKey);
- return sasKey;
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while "
- + "generating SAS key for Blob: " + relativePath + " inside "
- + "container : " + container + " in Storage Account : " + accountName,
- stoEx);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException "
- + "while generating SAS key for Blob: " + relativePath + " inside "
- + "container: " + container + " in Storage Account : " + accountName,
- uriSyntaxEx);
- }
- }
-
- /**
- * Helper method that creates CloudStorageAccount Instance using the
- * storage account key.
- * @param accountName Name of the storage account
- * @param accountKey Storage Account key
- * @return CloudStorageAccount instance for the storage account.
- * @throws SASKeyGenerationException
- */
- private CloudStorageAccount getStorageAccountInstance(String accountName,
- String accountKey) throws SASKeyGenerationException {
-
- if (!storageAccountMap.containsKey(accountName)) {
- if (accountKey == null || accountKey.isEmpty()) {
- throw new SASKeyGenerationException(
- "No key for Storage account " + accountName);
- }
-
- CloudStorageAccount account = null;
- try {
- account =
- new CloudStorageAccount(new StorageCredentialsAccountAndKey(
- accountName, accountKey));
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException "
- + "for account " + accountName, uriSyntaxEx);
- }
-
- storageAccountMap.put(accountName, account);
- }
-
- return storageAccountMap.get(accountName);
- }
-
- /**
- * Helper method that returns the Storage account name without
- * the domain name suffix.
- * @param fullAccountName Storage account name with domain name suffix
- * @return String
- */
- private String getAccountNameWithoutDomain(String fullAccountName) {
- StringTokenizer tokenizer = new StringTokenizer(fullAccountName, ".");
- return tokenizer.nextToken();
- }
-
- /**
- * Helper method to generate Access Policy for the Storage Account SAS Key
- * @return SharedAccessAccountPolicy
- */
- private SharedAccessAccountPolicy getDefaultAccountAccessPolicy() {
-
- SharedAccessAccountPolicy ap =
- new SharedAccessAccountPolicy();
-
- Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
- cal.setTime(new Date());
- cal.add(Calendar.HOUR, (int) getSasKeyExpiryPeriod() * HOURS_IN_DAY);
-
- ap.setSharedAccessExpiryTime(cal.getTime());
- ap.setPermissions(getDefaultAccoutSASKeyPermissions());
- ap.setResourceTypes(EnumSet.of(SharedAccessAccountResourceType.CONTAINER,
- SharedAccessAccountResourceType.OBJECT));
- ap.setServices(EnumSet.of(SharedAccessAccountService.BLOB));
-
- return ap;
- }
-
- private EnumSet<SharedAccessAccountPermissions> getDefaultAccoutSASKeyPermissions() {
- return EnumSet.of(SharedAccessAccountPermissions.ADD,
- SharedAccessAccountPermissions.CREATE,
- SharedAccessAccountPermissions.DELETE,
- SharedAccessAccountPermissions.LIST,
- SharedAccessAccountPermissions.READ,
- SharedAccessAccountPermissions.UPDATE,
- SharedAccessAccountPermissions.WRITE);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 4e777da8b40..36d732aeefc 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -18,4039 +18,131 @@
package org.apache.hadoop.fs.azure;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.TimeZone;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.Stack;
-import java.util.HashMap;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BufferedFSInputStream;
-import org.apache.hadoop.fs.CommonPathCapabilities;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PositionedReadable;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.fs.azure.security.RemoteWasbDelegationTokenManager;
-import org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager;
-import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
-import org.apache.hadoop.fs.impl.OpenFileParameters;
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.util.LambdaUtils;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Time;
-
-import org.apache.hadoop.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
-import static org.apache.hadoop.fs.azure.NativeAzureFileSystemHelper.*;
-import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.classification.VisibleForTesting;
-import com.microsoft.azure.storage.StorageException;
-
-/**
- * A {@link FileSystem} for reading and writing files stored on <a
- * href="http://store.azure.com/">Windows Azure</a>. This implementation is
- * blob-based and stores files on Azure in their native form so they can be read
- * by other Azure tools.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class NativeAzureFileSystem extends FileSystem {
- private static final int USER_WX_PERMISION = 0300;
- private static final String USER_HOME_DIR_PREFIX_DEFAULT = "/user";
- /**
- * A description of a folder rename operation, including the source and
- * destination keys, and descriptions of the files in the source folder.
- */
-
- public static class FolderRenamePending {
- private SelfRenewingLease folderLease;
- private String srcKey;
- private String dstKey;
- private FileMetadata[] fileMetadata = null; // descriptions of source files
- private ArrayList<String> fileStrings = null;
- private NativeAzureFileSystem fs;
- private static final int MAX_RENAME_PENDING_FILE_SIZE = 10000000;
- private static final int FORMATTING_BUFFER = 10000;
- private boolean committed;
- public static final String SUFFIX = "-RenamePending.json";
- private static final ObjectReader READER = new ObjectMapper()
- .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
- .readerFor(JsonNode.class);
-
- // Prepare in-memory information needed to do or redo a folder rename.
- public FolderRenamePending(String srcKey, String dstKey, SelfRenewingLease lease,
- NativeAzureFileSystem fs) throws IOException {
- this.srcKey = srcKey;
- this.dstKey = dstKey;
- this.folderLease = lease;
- this.fs = fs;
-
- // List all the files in the folder.
- long start = Time.monotonicNow();
- fileMetadata = fs.getStoreInterface().list(srcKey, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH);
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for rename operation is: {} ms", fileMetadata.length, (end - start));
-
- this.committed = true;
- }
-
- // Prepare in-memory information needed to do or redo folder rename from
- // a -RenamePending.json file read from storage. This constructor is to use during
- // redo processing.
- public FolderRenamePending(Path redoFile, NativeAzureFileSystem fs)
- throws IllegalArgumentException, IOException {
-
- this.fs = fs;
-
- // open redo file
- Path f = redoFile;
- int l;
- byte[] bytes;
- try (FSDataInputStream input = fs.open(f)) {
- bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
- l = input.read(bytes);
- }
- if (l <= 0) {
- // Jira HADOOP-12678 -Handle empty rename pending metadata file during
- // atomic rename in redo path. If during renamepending file is created
- // but not written yet, then this means that rename operation
- // has not started yet. So we should delete rename pending metadata file.
- LOG.error("Deleting empty rename pending file "
- + redoFile + " -- no data available");
- deleteRenamePendingFile(fs, redoFile);
- return;
- }
- if (l == MAX_RENAME_PENDING_FILE_SIZE) {
- throw new IOException(
- "Error reading pending rename file contents -- "
- + "maximum file size exceeded");
- }
- String contents = new String(bytes, 0, l, StandardCharsets.UTF_8);
-
- // parse the JSON
- JsonNode json = null;
- try {
- json = READER.readValue(contents);
- this.committed = true;
- } catch (JsonMappingException e) {
-
- // The -RedoPending.json file is corrupted, so we assume it was
- // not completely written
- // and the redo operation did not commit.
- this.committed = false;
- } catch (JsonParseException e) {
- this.committed = false;
- } catch (IOException e) {
- this.committed = false;
- }
-
- if (!this.committed) {
- LOG.error("Deleting corruped rename pending file {} \n {}",
- redoFile, contents);
-
- // delete the -RenamePending.json file
- deleteRenamePendingFile(fs, redoFile);
- return;
- }
-
- // initialize this object's fields
- ArrayList<String> fileStrList = new ArrayList<String>();
- JsonNode oldFolderName = json.get("OldFolderName");
- JsonNode newFolderName = json.get("NewFolderName");
- if (oldFolderName == null || newFolderName == null) {
- this.committed = false;
- } else {
- this.srcKey = oldFolderName.textValue();
- this.dstKey = newFolderName.textValue();
- if (this.srcKey == null || this.dstKey == null) {
- this.committed = false;
- } else {
- JsonNode fileList = json.get("FileList");
- if (fileList == null) {
- this.committed = false;
- } else {
- for (int i = 0; i < fileList.size(); i++) {
- fileStrList.add(fileList.get(i).textValue());
- }
- }
- }
- }
- this.fileStrings = fileStrList;
- }
-
- public FileMetadata[] getFiles() {
- return fileMetadata;
- }
-
- public SelfRenewingLease getFolderLease() {
- return folderLease;
- }
-
- /**
- * Deletes rename pending metadata file
- * @param fs -- the file system
- * @param redoFile - rename pending metadata file path
- * @throws IOException - If deletion fails
- */
- @VisibleForTesting
- void deleteRenamePendingFile(FileSystem fs, Path redoFile)
- throws IOException {
- try {
- fs.delete(redoFile, false);
- } catch (IOException e) {
- // If the rename metadata was not found then somebody probably
- // raced with us and finished the delete first
- Throwable t = e.getCause();
- if (t != null && t instanceof StorageException
- && "BlobNotFound".equals(((StorageException) t).getErrorCode())) {
- LOG.warn("rename pending file " + redoFile + " is already deleted");
- } else {
- throw e;
- }
- }
- }
-
- /**
- * Write to disk the information needed to redo folder rename,
- * in JSON format. The file name will be
- * {@code wasb://<sourceFolderPrefix>/folderName-RenamePending.json}
- * The file format will be:
- * <pre>{@code
- * {
- * FormatVersion: "1.0",
- * OperationTime: "<YYYY-MM-DD HH:MM:SS.MMM>",
- * OldFolderName: "<key>",
- * NewFolderName: "<key>",
- * FileList: [ <string> , <string> , ... ]
- * }
- *
- * Here's a sample:
- * {
- * FormatVersion: "1.0",
- * OperationUTCTime: "2014-07-01 23:50:35.572",
- * OldFolderName: "user/ehans/folderToRename",
- * NewFolderName: "user/ehans/renamedFolder",
- * FileList: [
- * "innerFile",
- * "innerFile2"
- * ]
- * } }</pre>
- * @param fs file system on which a file is written.
- * @throws IOException Thrown when fail to write file.
- */
- public void writeFile(NativeAzureFileSystem fs) throws IOException {
- Path path = getRenamePendingFilePath();
- LOG.debug("Preparing to write atomic rename state to {}", path.toString());
- OutputStream output = null;
-
- String contents = makeRenamePendingFileContents();
-
- // Write file.
- try {
- output = fs.createInternal(path, FsPermission.getFileDefault(), false, null);
- output.write(contents.getBytes(StandardCharsets.UTF_8));
- } catch (IOException e) {
- throw new IOException("Unable to write RenamePending file for folder rename from "
- + srcKey + " to " + dstKey, e);
- } finally {
- NativeAzureFileSystemHelper.cleanup(LOG, output);
- }
- }
-
- /**
- * Return the contents of the JSON file to represent the operations
- * to be performed for a folder rename.
- *
- * @return JSON string which represents the operation.
- */
- public String makeRenamePendingFileContents() {
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
- sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
- String time = sdf.format(new Date());
-
- // Make file list string
- StringBuilder builder = new StringBuilder();
- builder.append("[\n");
- for (int i = 0; i != fileMetadata.length; i++) {
- if (i > 0) {
- builder.append(",\n");
- }
- builder.append(" ");
- String noPrefix = StringUtils.removeStart(fileMetadata[i].getKey(), srcKey + "/");
-
- // Quote string file names, escaping any possible " characters or other
- // necessary characters in the name.
- builder.append(quote(noPrefix));
- if (builder.length() >=
- MAX_RENAME_PENDING_FILE_SIZE - FORMATTING_BUFFER) {
-
- // Give up now to avoid using too much memory.
- LOG.error("Internal error: Exceeded maximum rename pending file size of {} bytes.",
- MAX_RENAME_PENDING_FILE_SIZE);
-
- // return some bad JSON with an error message to make it human readable
- return "exceeded maximum rename pending file size";
- }
- }
- builder.append("\n ]");
- String fileList = builder.toString();
-
- // Make file contents as a string. Again, quote file names, escaping
- // characters as appropriate.
- String contents = "{\n"
- + " FormatVersion: \"1.0\",\n"
- + " OperationUTCTime: \"" + time + "\",\n"
- + " OldFolderName: " + quote(srcKey) + ",\n"
- + " NewFolderName: " + quote(dstKey) + ",\n"
- + " FileList: " + fileList + "\n"
- + "}\n";
-
- return contents;
- }
-
- /**
- * This is an exact copy of org.codehaus.jettison.json.JSONObject.quote
- * method.
- *
- * Produce a string in double quotes with backslash sequences in all the
- * right places. A backslash will be inserted within </, allowing JSON
- * text to be delivered in HTML. In JSON text, a string cannot contain a
- * control character or an unescaped quote or backslash.
- * @param string A String
- * @return A String correctly formatted for insertion in a JSON text.
- */
- private String quote(String string) {
- if (string == null || string.length() == 0) {
- return "\"\"";
- }
-
- char c = 0;
- int i;
- int len = string.length();
- StringBuilder sb = new StringBuilder(len + 4);
- String t;
-
- sb.append('"');
- for (i = 0; i < len; i += 1) {
- c = string.charAt(i);
- switch (c) {
- case '\\':
- case '"':
- sb.append('\\');
- sb.append(c);
- break;
- case '/':
- sb.append('\\');
- sb.append(c);
- break;
- case '\b':
- sb.append("\\b");
- break;
- case '\t':
- sb.append("\\t");
- break;
- case '\n':
- sb.append("\\n");
- break;
- case '\f':
- sb.append("\\f");
- break;
- case '\r':
- sb.append("\\r");
- break;
- default:
- if (c < ' ') {
- t = "000" + Integer.toHexString(c);
- sb.append("\\u" + t.substring(t.length() - 4));
- } else {
- sb.append(c);
- }
- }
- }
- sb.append('"');
- return sb.toString();
- }
-
- public String getSrcKey() {
- return srcKey;
- }
-
- public String getDstKey() {
- return dstKey;
- }
-
- public FileMetadata getSourceMetadata() throws IOException {
- return fs.getStoreInterface().retrieveMetadata(srcKey);
- }
-
- /**
- * Execute a folder rename. This is the execution path followed
- * when everything is working normally. See redo() for the alternate
- * execution path for the case where we're recovering from a folder rename
- * failure.
- * @throws IOException Thrown when fail to renaming.
- */
- public void execute() throws IOException {
-
- AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
- @Override
- public boolean execute(FileMetadata file) throws IOException{
- renameFile(file);
- return true;
- }
- };
-
- AzureFileSystemThreadPoolExecutor executor = this.fs.getThreadPoolExecutor(this.fs.renameThreadCount,
- "AzureBlobRenameThread", "Rename", getSrcKey(), AZURE_RENAME_THREADS);
-
- executor.executeParallel(this.getFiles(), task);
-
- // Rename the source folder 0-byte root file itself.
- FileMetadata srcMetadata2 = this.getSourceMetadata();
- if (srcMetadata2.getBlobMaterialization() ==
- BlobMaterialization.Explicit) {
-
- // It already has a lease on it from the "prepare" phase so there's no
- // need to get one now. Pass in existing lease to allow file delete.
- fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(),
- false, folderLease);
- }
-
- // Update the last-modified time of the parent folders of both source and
- // destination.
- fs.updateParentFolderLastModifiedTime(srcKey);
- fs.updateParentFolderLastModifiedTime(dstKey);
- }
-
- // Rename a single file
- @VisibleForTesting
- void renameFile(FileMetadata file) throws IOException{
- // Rename all materialized entries under the folder to point to the
- // final destination.
- if (file.getBlobMaterialization() == BlobMaterialization.Explicit) {
- String srcName = file.getKey();
- String suffix = srcName.substring((this.getSrcKey()).length());
- String dstName = this.getDstKey() + suffix;
-
- // Rename gets exclusive access (via a lease) for files
- // designated for atomic rename.
- // The main use case is for HBase write-ahead log (WAL) and data
- // folder processing correctness. See the rename code for details.
- boolean acquireLease = this.fs.getStoreInterface().isAtomicRenameKey(srcName);
- this.fs.getStoreInterface().rename(srcName, dstName, acquireLease, null);
- }
- }
-
- /** Clean up after execution of rename.
- * @throws IOException Thrown when fail to clean up.
- * */
- public void cleanup() throws IOException {
-
- if (fs.getStoreInterface().isAtomicRenameKey(srcKey)) {
-
- // Remove RenamePending file
- fs.delete(getRenamePendingFilePath(), false);
-
- // Freeing source folder lease is not necessary since the source
- // folder file was deleted.
- }
- }
-
- private Path getRenamePendingFilePath() {
- String fileName = srcKey + SUFFIX;
- Path fileNamePath = keyToPath(fileName);
- Path path = fs.makeAbsolute(fileNamePath);
- return path;
- }
-
- /**
- * Recover from a folder rename failure by redoing the intended work,
- * as recorded in the -RenamePending.json file.
- *
- * @throws IOException Thrown when fail to redo.
- */
- public void redo() throws IOException {
-
- if (!committed) {
-
- // Nothing to do. The -RedoPending.json file should have already been
- // deleted.
- return;
- }
-
- // Try to get a lease on source folder to block concurrent access to it.
- // It may fail if the folder is already gone. We don't check if the
- // source exists explicitly because that could recursively trigger redo
- // and give an infinite recursion.
- SelfRenewingLease lease = null;
- boolean sourceFolderGone = false;
- try {
- lease = fs.leaseSourceFolder(srcKey);
- } catch (AzureException e) {
-
- // If the source folder was not found then somebody probably
- // raced with us and finished the rename first, or the
- // first rename failed right before deleting the rename pending
- // file.
- String errorCode = "";
- try {
- StorageException se = (StorageException) e.getCause();
- errorCode = se.getErrorCode();
- } catch (Exception e2) {
- ; // do nothing -- could not get errorCode
- }
- if (errorCode.equals("BlobNotFound")) {
- sourceFolderGone = true;
- } else {
- throw new IOException(
- "Unexpected error when trying to lease source folder name during "
- + "folder rename redo",
- e);
- }
- }
-
- if (!sourceFolderGone) {
- // Make sure the target folder exists.
- Path dst = fullPath(dstKey);
- if (!fs.existsInternal(dst)) {
- fs.mkdirs(dst);
- }
-
- // For each file inside the folder to be renamed,
- // make sure it has been renamed.
- for(String fileName : fileStrings) {
- finishSingleFileRename(fileName);
- }
-
- // Remove the source folder. Don't check explicitly if it exists,
- // to avoid triggering redo recursively.
- try {
- // Rename the source folder 0-byte root file
- // as destination folder 0-byte root file.
- FileMetadata srcMetaData = this.getSourceMetadata();
- if (srcMetaData.getBlobMaterialization() == BlobMaterialization.Explicit) {
- // We already have a lease. So let's just rename the source blob
- // as destination blob under same lease.
- fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(), false, lease);
- }
-
- // Now we can safely delete the source folder.
- fs.getStoreInterface().delete(srcKey, lease);
- } catch (Exception e) {
- LOG.info("Unable to delete source folder during folder rename redo. "
- + "If the source folder is already gone, this is not an error "
- + "condition. Continuing with redo.", e);
- }
-
- // Update the last-modified time of the parent folders of both source
- // and destination.
- fs.updateParentFolderLastModifiedTime(srcKey);
- fs.updateParentFolderLastModifiedTime(dstKey);
- }
-
- // Remove the -RenamePending.json file.
- fs.delete(getRenamePendingFilePath(), false);
- }
-
- // See if the source file is still there, and if it is, rename it.
- private void finishSingleFileRename(String fileName)
- throws IOException {
- Path srcFile = fullPath(srcKey, fileName);
- Path dstFile = fullPath(dstKey, fileName);
- String srcName = fs.pathToKey(srcFile);
- String dstName = fs.pathToKey(dstFile);
- boolean srcExists = fs.getStoreInterface().explicitFileExists(srcName);
- boolean dstExists = fs.getStoreInterface().explicitFileExists(dstName);
- if(srcExists) {
- // Rename gets exclusive access (via a lease) for HBase write-ahead log
- // (WAL) file processing correctness. See the rename code for details.
- fs.getStoreInterface().rename(srcName, dstName, true, null);
- } else if (!srcExists && dstExists) {
- // The rename already finished, so do nothing.
- ;
- } else {
- // HADOOP-14512
- LOG.warn(
- "Attempting to complete rename of file " + srcKey + "/" + fileName
- + " during folder rename redo, and file was not found in source "
- + "or destination " + dstKey + "/" + fileName + ". "
- + "This must mean the rename of this file has already completed");
- }
- }
-
- // Return an absolute path for the specific fileName within the folder
- // specified by folderKey.
- private Path fullPath(String folderKey, String fileName) {
- return new Path(new Path(fs.getUri()), "/" + folderKey + "/" + fileName);
- }
-
- private Path fullPath(String fileKey) {
- return new Path(new Path(fs.getUri()), "/" + fileKey);
- }
- }
-
- private static final String TRAILING_PERIOD_PLACEHOLDER = "[[.]]";
- private static final Pattern TRAILING_PERIOD_PLACEHOLDER_PATTERN =
- Pattern.compile("\\[\\[\\.\\]\\](?=$|/)");
- private static final Pattern TRAILING_PERIOD_PATTERN = Pattern.compile("\\.(?=$|/)");
-
- @Override
- public String getScheme() {
- return "wasb";
- }
-
- /**
- * If fs.azure.override.canonical.service.name is set as true, return URI of
- * the WASB filesystem, otherwise use the default implementation.
- *
- * @return a service string that uniquely identifies this file system
- */
- @Override
- public String getCanonicalServiceName() {
- if (returnUriAsCanonicalServiceName) {
- return getUri().toString();
- }
- return super.getCanonicalServiceName();
- }
-
-
- /**
- * <p>
- * A {@link FileSystem} for reading and writing files stored on <a
- * href="http://store.azure.com/">Windows Azure</a>. This implementation is
- * blob-based and stores files on Azure in their native form so they can be read
- * by other Azure tools. This implementation uses HTTPS for secure network communication.
- * </p>
- */
- public static class Secure extends NativeAzureFileSystem {
- @Override
- public String getScheme() {
- return "wasbs";
- }
- }
-
- public static final Logger LOG = LoggerFactory.getLogger(NativeAzureFileSystem.class);
-
- /**
- * The time span in seconds before which we consider a temp blob to be
- * dangling (not being actively uploaded to) and up for reclamation.
- *
- * So e.g. if this is 60, then any temporary blobs more than a minute old
- * would be considered dangling.
- */
- static final String AZURE_TEMP_EXPIRY_PROPERTY_NAME = "fs.azure.fsck.temp.expiry.seconds";
- private static final int AZURE_TEMP_EXPIRY_DEFAULT = 3600;
- static final String PATH_DELIMITER = Path.SEPARATOR;
- static final String AZURE_TEMP_FOLDER = "_$azuretmpfolder$";
-
- private static final int AZURE_LIST_ALL = -1;
- private static final int AZURE_UNBOUNDED_DEPTH = -1;
-
- /**
- * The configuration property that determines which group owns files created
- * in WASB.
- */
- private static final String AZURE_DEFAULT_GROUP_PROPERTY_NAME = "fs.azure.permissions.supergroup";
- /**
- * The default value for fs.azure.permissions.supergroup. Chosen as the same
- * default as DFS.
- */
- static final String AZURE_DEFAULT_GROUP_DEFAULT = "supergroup";
-
- /**
- * Configuration property used to specify list of users that can perform
- * chown operation when authorization is enabled in WASB.
- */
- public static final String AZURE_CHOWN_USERLIST_PROPERTY_NAME =
- "fs.azure.chown.allowed.userlist";
-
- static final String AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
-
- /**
- * Configuration property used to specify list of daemon users that can
- * perform chmod operation when authorization is enabled in WASB.
- */
- public static final String AZURE_DAEMON_USERLIST_PROPERTY_NAME =
- "fs.azure.daemon.userlist";
-
- static final String AZURE_DAEMON_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
-
- /**
- * Configuration property used to specify list of users that can perform
- * chmod operation when authorization is enabled in WASB.
- */
- public static final String AZURE_CHMOD_USERLIST_PROPERTY_NAME =
- "fs.azure.chmod.allowed.userlist";
-
- static final String AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
-
- static final String AZURE_RINGBUFFER_CAPACITY_PROPERTY_NAME =
- "fs.azure.ring.buffer.capacity";
- static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME =
- "fs.azure.output.stream.buffer.size";
-
- public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics";
-
- /*
- * Property to enable Append API.
- */
- public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support";
-
- /*
- * Property to override canonical service name with filesystem's URI.
- */
- public static final String RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME = "fs.azure.override.canonical.service.name";
-
- /**
- * The configuration property to set number of threads to be used for rename operation.
- */
- public static final String AZURE_RENAME_THREADS = "fs.azure.rename.threads";
-
- /**
- * The default number of threads to be used for rename operation.
- */
- public static final int DEFAULT_AZURE_RENAME_THREADS = 0;
-
- /**
- * The configuration property to set number of threads to be used for delete operation.
- */
- public static final String AZURE_DELETE_THREADS = "fs.azure.delete.threads";
-
- /**
- * The default number of threads to be used for delete operation.
- */
- public static final int DEFAULT_AZURE_DELETE_THREADS = 0;
-
- /**
- * The number of threads to be used for delete operation after reading user configuration.
- */
- private int deleteThreadCount = 0;
-
- /**
- * The number of threads to be used for rename operation after reading user configuration.
- */
- private int renameThreadCount = 0;
-
- private class NativeAzureFsInputStream extends FSInputStream {
- private InputStream in;
- private final String key;
- private long pos = 0;
- private boolean closed = false;
- private boolean isPageBlob;
-
- // File length, valid only for streams over block blobs.
- private long fileLength;
-
- NativeAzureFsInputStream(InputStream in, String key, long fileLength) {
- this.in = in;
- this.key = key;
- this.isPageBlob = store.isPageBlobKey(key);
- this.fileLength = fileLength;
- }
-
- /**
- * Return the size of the remaining available bytes
- * if the size is less than or equal to {@link Integer#MAX_VALUE},
- * otherwise, return {@link Integer#MAX_VALUE}.
- *
- * This is to match the behavior of DFSInputStream.available(),
- * which some clients may rely on (HBase write-ahead log reading in
- * particular).
- */
- @Override
- public synchronized int available() throws IOException {
- if (isPageBlob) {
- return in.available();
- } else {
- if (closed) {
- throw new IOException("Stream closed");
- }
- final long remaining = this.fileLength - pos;
- return remaining <= Integer.MAX_VALUE ?
- (int) remaining : Integer.MAX_VALUE;
- }
- }
-
- /*
- * Reads the next byte of data from the input stream. The value byte is
- * returned as an integer in the range 0 to 255. If no byte is available
- * because the end of the stream has been reached, the value -1 is returned.
- * This method blocks until input data is available, the end of the stream
- * is detected, or an exception is thrown.
- *
- * @returns int An integer corresponding to the byte read.
- */
- @Override
- public synchronized int read() throws FileNotFoundException, IOException {
- try {
- int result = 0;
- result = in.read();
- if (result != -1) {
- pos++;
- if (statistics != null) {
- statistics.incrementBytesRead(1);
- }
- }
- // Return to the caller with the result.
- //
- return result;
- } catch(EOFException e) {
- return -1;
- } catch(IOException e) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
-
- LOG.error("Encountered Storage Exception for read on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e, ((StorageException) innerException).getErrorCode());
-
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
- }
-
- throw e;
- }
- }
-
- /*
- * Reads up to len bytes of data from the input stream into an array of
- * bytes. An attempt is made to read as many as len bytes, but a smaller
- * number may be read. The number of bytes actually read is returned as an
- * integer. This method blocks until input data is available, end of file is
- * detected, or an exception is thrown. If len is zero, then no bytes are
- * read and 0 is returned; otherwise, there is an attempt to read at least
- * one byte. If no byte is available because the stream is at end of file,
- * the value -1 is returned; otherwise, at least one byte is read and stored
- * into b.
- *
- * @param b -- the buffer into which data is read
- *
- * @param off -- the start offset in the array b at which data is written
- *
- * @param len -- the maximum number of bytes read
- *
- * @ returns int The total number of byes read into the buffer, or -1 if
- * there is no more data because the end of stream is reached.
- */
- @Override
- public synchronized int read(byte[] b, int off, int len) throws FileNotFoundException, IOException {
- try {
- int result = 0;
- result = in.read(b, off, len);
- if (result > 0) {
- pos += result;
- }
-
- if (null != statistics && result > 0) {
- statistics.incrementBytesRead(result);
- }
-
- // Return to the caller with the result.
- return result;
- } catch(IOException e) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
-
- LOG.error("Encountered Storage Exception for read on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e, ((StorageException) innerException).getErrorCode());
-
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
- }
-
- throw e;
- }
- }
-
- @Override
- public int read(long position, byte[] buffer, int offset, int length)
- throws IOException {
- // SpotBugs reports bug type IS2_INCONSISTENT_SYNC here.
- // This report is not valid here.
- // 'this.in' is instance of BlockBlobInputStream and read(long, byte[], int, int)
- // calls it's Super class method when 'fs.azure.block.blob.buffered.pread.disable'
- // is configured false. Super class FSInputStream's implementation is having
- // proper synchronization.
- // When 'fs.azure.block.blob.buffered.pread.disable' is true, we want a lock free
- // implementation of blob read. Here we don't use any of the InputStream's
- // shared resource (buffer) and also don't change any cursor position etc.
- // So its safe to go with unsynchronized way of read.
- if (in instanceof PositionedReadable) {
- try {
- int result = ((PositionedReadable) this.in).read(position, buffer,
- offset, length);
- if (null != statistics && result > 0) {
- statistics.incrementBytesRead(result);
- }
- return result;
- } catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
- if (innerException instanceof StorageException) {
- LOG.error("Encountered Storage Exception for read on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e, ((StorageException) innerException).getErrorCode());
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
- }
- throw e;
- }
- }
- return super.read(position, buffer, offset, length);
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (!closed) {
- closed = true;
- IOUtils.closeStream(in);
- in = null;
- }
- }
-
- @Override
- public synchronized void seek(long pos) throws FileNotFoundException, EOFException, IOException {
- try {
- checkNotClosed();
- if (pos < 0) {
- throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
- }
- if (this.pos > pos) {
- if (in instanceof Seekable) {
- ((Seekable) in).seek(pos);
- this.pos = pos;
- } else {
- IOUtils.closeStream(in);
- in = store.retrieve(key);
- this.pos = in.skip(pos);
- }
- } else {
- this.pos += in.skip(pos - this.pos);
- }
- LOG.debug("Seek to position {}. Bytes skipped {}", pos,
- this.pos);
- } catch(IOException e) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
-
- throw e;
- } catch(IndexOutOfBoundsException e) {
- throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
- }
- }
-
- @Override
- public synchronized long getPos() throws IOException {
- return pos;
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
-
- /*
- * Helper method to check if a stream is closed.
- */
- private void checkNotClosed() throws IOException {
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
- }
- }
-
- /**
- * Azure output stream; wraps an inner stream of different types.
- */
- public class NativeAzureFsOutputStream extends OutputStream
- implements Syncable, StreamCapabilities {
- private String key;
- private String keyEncoded;
- private OutputStream out;
-
- public NativeAzureFsOutputStream(OutputStream out, String aKey,
- String anEncodedKey) throws IOException {
- // Check input arguments. The output stream should be non-null and the
- // keys
- // should be valid strings.
- if (null == out) {
- throw new IllegalArgumentException(
- "Illegal argument: the output stream is null.");
- }
-
- if (null == aKey || 0 == aKey.length()) {
- throw new IllegalArgumentException(
- "Illegal argument the key string is null or empty");
- }
-
- if (null == anEncodedKey || 0 == anEncodedKey.length()) {
- throw new IllegalArgumentException(
- "Illegal argument the encoded key string is null or empty");
- }
-
- // Initialize the member variables with the incoming parameters.
- this.out = out;
-
- setKey(aKey);
- setEncodedKey(anEncodedKey);
- }
-
- /**
- * Get a reference to the wrapped output stream.
- *
- * @return the underlying output stream
- */
- @InterfaceAudience.LimitedPrivate({"HDFS"})
- public OutputStream getOutStream() {
- return out;
- }
-
- @Override // Syncable
- public void hflush() throws IOException {
- if (out instanceof Syncable) {
- ((Syncable) out).hflush();
- } else {
- flush();
- }
- }
-
- @Override // Syncable
- public void hsync() throws IOException {
- if (out instanceof Syncable) {
- ((Syncable) out).hsync();
- } else {
- flush();
- }
- }
-
- /**
- * Propagate probe of stream capabilities to nested stream
- * (if supported), else return false.
- * @param capability string to query the stream support for.
- * @return true if the nested stream supports the specific capability.
- */
- @Override // StreamCapability
- public boolean hasCapability(String capability) {
- return StoreImplementationUtils.hasCapability(out, capability);
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (out != null) {
- // Close the output stream and decode the key for the output stream
- // before returning to the caller.
- //
- out.close();
- try {
- restoreKey();
- } finally {
- out = null;
- }
- }
- }
-
- /**
- * Writes the specified byte to this output stream. The general contract for
- * write is that one byte is written to the output stream. The byte to be
- * written is the eight low-order bits of the argument b. The 24 high-order
- * bits of b are ignored.
- *
- * @param b
- * 32-bit integer of block of 4 bytes
- */
- @Override
- public void write(int b) throws IOException {
- checkOpen();
- try {
- out.write(b);
- } catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
- LOG.error("Encountered Storage Exception for write on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
- }
- throw e;
- }
- }
-
- /**
- * Writes b.length bytes from the specified byte array to this output
- * stream. The general contract for write(b) is that it should have exactly
- * the same effect as the call write(b, 0, b.length).
- *
- * @param b
- * Block of bytes to be written to the output stream.
- */
- @Override
- public void write(byte[] b) throws IOException {
- checkOpen();
- try {
- out.write(b);
- } catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
- LOG.error("Encountered Storage Exception for write on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
- }
- throw e;
- }
- }
-
- /**
- * Writes <code>len</code> from the specified byte array starting at offset
- * <code>off</code> to the output stream. The general contract for write(b,
- * off, len) is that some of the bytes in the array <code>b</code>
- * are written to the output stream in order; element <code>b[off]</code>
- * is the first byte written and <code>b[off+len-1]</code> is the last
- * byte written by this operation.
- *
- * @param b
- * Byte array to be written.
- * @param off
- * Write this offset in stream.
- * @param len
- * Number of bytes to be written.
- */
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- checkOpen();
- try {
- out.write(b, off, len);
- } catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
- LOG.error("Encountered Storage Exception for write on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
- }
- throw e;
- }
- }
-
- /**
- * Get the blob name.
- *
- * @return String Blob name.
- */
- public String getKey() {
- return key;
- }
-
- /**
- * Set the blob name.
- *
- * @param key
- * Blob name.
- */
- public void setKey(String key) {
- this.key = key;
- }
-
- /**
- * Get the blob name.
- *
- * @return String Blob name.
- */
- public String getEncodedKey() {
- return keyEncoded;
- }
-
- /**
- * Set the blob name.
- *
- * @param anEncodedKey
- * Blob name.
- */
- public void setEncodedKey(String anEncodedKey) {
- this.keyEncoded = anEncodedKey;
- }
-
- /**
- * Restore the original key name from the m_key member variable. Note: The
- * output file stream is created with an encoded blob store key to guarantee
- * load balancing on the front end of the Azure storage partition servers.
- * The create also includes the name of the original key value which is
- * stored in the m_key member variable. This method should only be called
- * when the stream is closed.
- */
- private void restoreKey() throws IOException {
- store.rename(getEncodedKey(), getKey());
- }
-
- /**
- * Check for the stream being open.
- * @throws IOException if the stream is closed.
- */
- private void checkOpen() throws IOException {
- if (out == null) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
- }
-
- }
-
- private URI uri;
- private NativeFileSystemStore store;
- private AzureNativeFileSystemStore actualStore;
- private Path workingDir;
- private AzureFileSystemInstrumentation instrumentation;
- private String metricsSourceName;
- private boolean isClosed = false;
- private static boolean suppressRetryPolicy = false;
- // A counter to create unique (within-process) names for my metrics sources.
- private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
- private boolean appendSupportEnabled = false;
- private boolean returnUriAsCanonicalServiceName = false;
- private DelegationTokenAuthenticatedURL authURL;
- private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
- private String credServiceUrl;
- private List<String> chownAllowedUsers;
- private List<String> chmodAllowedUsers;
- private List<String> daemonUsers;
- /**
- * Configuration key to enable authorization support in WASB.
- */
- public static final String KEY_AZURE_AUTHORIZATION =
- "fs.azure.authorization";
-
- /**
- * Default value for the authorization support in WASB.
- */
- private static final boolean DEFAULT_AZURE_AUTHORIZATION = false;
-
- /**
- * Flag controlling authorization support in WASB.
- */
- private boolean azureAuthorization = false;
-
- /**
- * Flag controlling Kerberos support in WASB.
- */
- private boolean kerberosSupportEnabled = false;
-
- /**
- * Authorizer to use when authorization support is enabled in
- * WASB.
- */
- private WasbAuthorizerInterface authorizer = null;
-
- private UserGroupInformation ugi;
-
- private WasbDelegationTokenManager wasbDelegationTokenManager;
-
- public NativeAzureFileSystem() {
- // set store in initialize()
- }
-
- public NativeAzureFileSystem(NativeFileSystemStore store) {
- this.store = store;
- }
-
- /**
- * Suppress the default retry policy for the Storage, useful in unit tests to
- * test negative cases without waiting forever.
- */
- @VisibleForTesting
- static void suppressRetryPolicy() {
- suppressRetryPolicy = true;
- }
-
- /**
- * Undo the effect of suppressRetryPolicy.
- */
- @VisibleForTesting
- static void resumeRetryPolicy() {
- suppressRetryPolicy = false;
- }
-
- /**
- * Creates a new metrics source name that's unique within this process.
- * @return metric source name
- */
- @VisibleForTesting
- public static String newMetricsSourceName() {
- int number = metricsSourceNameCounter.incrementAndGet();
- final String baseName = "AzureFileSystemMetrics";
- if (number == 1) { // No need for a suffix for the first one
- return baseName;
- } else {
- return baseName + number;
- }
- }
-
- /**
- * Checks if the given URI scheme is a scheme that's affiliated with the Azure
- * File System.
- *
- * @param scheme
- * The URI scheme.
- * @return true iff it's an Azure File System URI scheme.
- */
- private static boolean isWasbScheme(String scheme) {
- // The valid schemes are: asv (old name), asvs (old name over HTTPS),
- // wasb (new name), wasbs (new name over HTTPS).
- return scheme != null
- && (scheme.equalsIgnoreCase("asv") || scheme.equalsIgnoreCase("asvs")
- || scheme.equalsIgnoreCase("wasb") || scheme
- .equalsIgnoreCase("wasbs"));
- }
-
- /**
- * Puts in the authority of the default file system if it is a WASB file
- * system and the given URI's authority is null.
- *
- * @return The URI with reconstructed authority if necessary and possible.
- */
- private static URI reconstructAuthorityIfNeeded(URI uri, Configuration conf) {
- if (null == uri.getAuthority()) {
- // If WASB is the default file system, get the authority from there
- URI defaultUri = FileSystem.getDefaultUri(conf);
- if (defaultUri != null && isWasbScheme(defaultUri.getScheme())) {
- try {
- // Reconstruct the URI with the authority from the default URI.
- return new URI(uri.getScheme(), defaultUri.getAuthority(),
- uri.getPath(), uri.getQuery(), uri.getFragment());
- } catch (URISyntaxException e) {
- // This should never happen.
- throw new Error("Bad URI construction", e);
- }
- }
- }
- return uri;
- }
-
- @Override
- protected void checkPath(Path path) {
- // Make sure to reconstruct the path's authority if needed
- super.checkPath(new Path(reconstructAuthorityIfNeeded(path.toUri(),
- getConf())));
- }
-
- @Override
- public void initialize(URI uri, Configuration conf)
- throws IOException, IllegalArgumentException {
- // Check authority for the URI to guarantee that it is non-null.
- uri = reconstructAuthorityIfNeeded(uri, conf);
- if (null == uri.getAuthority()) {
- final String errMsg = String
- .format("Cannot initialize WASB file system, URI authority not recognized.");
- throw new IllegalArgumentException(errMsg);
- }
- super.initialize(uri, conf);
-
- if (store == null) {
- store = createDefaultStore(conf);
- }
-
- instrumentation = new AzureFileSystemInstrumentation(conf);
- if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
- // Make sure the metrics system is available before interacting with Azure
- AzureFileSystemMetricsSystem.fileSystemStarted();
- metricsSourceName = newMetricsSourceName();
- String sourceDesc = "Azure Storage Volume File System metrics";
- AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
- instrumentation);
- }
-
- store.initialize(uri, conf, instrumentation);
- setConf(conf);
- this.ugi = UserGroupInformation.getCurrentUser();
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser()
- .getShortUserName()).makeQualified(getUri(), getWorkingDirectory());
-
- this.appendSupportEnabled = conf.getBoolean(APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false);
- LOG.debug("NativeAzureFileSystem. Initializing.");
- LOG.debug(" blockSize = {}", store.getHadoopBlockSize());
-
- // Initialize thread counts from user configuration
- deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
- renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS);
-
- boolean useSecureMode = conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE,
- AzureNativeFileSystemStore.DEFAULT_USE_SECURE_MODE);
-
- this.azureAuthorization = useSecureMode &&
- conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION);
- this.kerberosSupportEnabled =
- conf.getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
-
- if (this.azureAuthorization) {
-
- this.authorizer =
- new RemoteWasbAuthorizerImpl();
- authorizer.init(conf);
-
- this.chmodAllowedUsers =
- Arrays.asList(conf.getTrimmedStrings(
- AZURE_CHMOD_USERLIST_PROPERTY_NAME,
- AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE));
- this.chownAllowedUsers =
- Arrays.asList(conf.getTrimmedStrings(
- AZURE_CHOWN_USERLIST_PROPERTY_NAME,
- AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE));
- this.daemonUsers =
- Arrays.asList(conf.getTrimmedStrings(
- AZURE_DAEMON_USERLIST_PROPERTY_NAME,
- AZURE_DAEMON_USERLIST_PROPERTY_DEFAULT_VALUE));
- }
-
- if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
- this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf);
- }
-
- this.returnUriAsCanonicalServiceName = conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
- }
-
- @Override
- public Path getHomeDirectory() {
- return makeQualified(new Path(
- USER_HOME_DIR_PREFIX_DEFAULT + "/" + this.ugi.getShortUserName()));
- }
-
- @VisibleForTesting
- public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) {
- this.authorizer = authorizer;
- }
-
- private NativeFileSystemStore createDefaultStore(Configuration conf) {
- actualStore = new AzureNativeFileSystemStore();
-
- if (suppressRetryPolicy) {
- actualStore.suppressRetryPolicy();
- }
- return actualStore;
- }
-
- /**
- * Azure Storage doesn't allow the blob names to end in a period,
- * so encode this here to work around that limitation.
- */
- private static String encodeTrailingPeriod(String toEncode) {
- Matcher matcher = TRAILING_PERIOD_PATTERN.matcher(toEncode);
- return matcher.replaceAll(TRAILING_PERIOD_PLACEHOLDER);
- }
-
- /**
- * Reverse the encoding done by encodeTrailingPeriod().
- */
- private static String decodeTrailingPeriod(String toDecode) {
- Matcher matcher = TRAILING_PERIOD_PLACEHOLDER_PATTERN.matcher(toDecode);
- return matcher.replaceAll(".");
- }
-
- /**
- * Convert the path to a key. By convention, any leading or trailing slash is
- * removed, except for the special case of a single slash.
- * @param path path converted to a key
- * @return key string
- */
- @VisibleForTesting
- public String pathToKey(Path path) {
- // Convert the path to a URI to parse the scheme, the authority, and the
- // path from the path object.
- URI tmpUri = path.toUri();
- String pathUri = tmpUri.getPath();
-
- // The scheme and authority is valid. If the path does not exist add a "/"
- // separator to list the root of the container.
- Path newPath = path;
- if ("".equals(pathUri)) {
- newPath = new Path(tmpUri.toString() + Path.SEPARATOR);
- }
-
- // Verify path is absolute if the path refers to a windows drive scheme.
- if (!newPath.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
-
- String key = null;
- key = newPath.toUri().getPath();
- key = removeTrailingSlash(key);
- key = encodeTrailingPeriod(key);
- if (key.length() == 1) {
- return key;
- } else {
- return key.substring(1); // remove initial slash
- }
- }
-
- // Remove any trailing slash except for the case of a single slash.
- private static String removeTrailingSlash(String key) {
- if (key.length() == 0 || key.length() == 1) {
- return key;
- }
- if (key.charAt(key.length() - 1) == '/') {
- return key.substring(0, key.length() - 1);
- } else {
- return key;
- }
- }
-
- static Path keyToPath(String key) {
- if (key.equals("/")) {
- return new Path("/"); // container
- }
- return new Path("/" + decodeTrailingPeriod(key));
- }
-
- /**
- * Get the absolute version of the path (fully qualified).
- * This is public for testing purposes.
- *
- * @param path path to be absolute path.
- * @return fully qualified path
- */
- @VisibleForTesting
- public Path makeAbsolute(Path path) {
- if (path.isAbsolute()) {
- return path;
- }
- return new Path(workingDir, path);
- }
-
- /**
- * For unit test purposes, retrieves the AzureNativeFileSystemStore store
- * backing this file system.
- *
- * @return The store object.
- */
- @VisibleForTesting
- public AzureNativeFileSystemStore getStore() {
- return actualStore;
- }
-
- NativeFileSystemStore getStoreInterface() {
- return store;
- }
-
- /**
- * @param requestingAccessForPath - The path to the ancestor/parent/subtree/file that needs to be
- * checked before granting access to originalPath
- * @param accessType - The type of access READ/WRITE being requested
- * @param operation - A string describing the operation being performed ("delete", "create" etc.).
- * @param originalPath - The originalPath that was being accessed
- */
- private void performAuthCheck(Path requestingAccessForPath, WasbAuthorizationOperations accessType,
- String operation, Path originalPath) throws WasbAuthorizationException, IOException {
-
- if (azureAuthorization && this.authorizer != null) {
-
- requestingAccessForPath = requestingAccessForPath.makeQualified(getUri(), getWorkingDirectory());
- originalPath = originalPath.makeQualified(getUri(), getWorkingDirectory());
-
- String owner = getOwnerForPath(requestingAccessForPath);
-
- if (!this.authorizer.authorize(requestingAccessForPath.toString(), accessType.toString(), owner)) {
- throw new WasbAuthorizationException(operation
- + " operation for Path : " + originalPath.toString() + " not allowed");
- }
-
- }
- }
-
- /**
- * Gets the metrics source for this file system.
- * This is mainly here for unit testing purposes.
- *
- * @return the metrics source.
- */
- public AzureFileSystemInstrumentation getInstrumentation() {
- return instrumentation;
- }
-
- /** This optional operation is not yet supported. */
- @Override
- public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
- throws IOException {
-
- if (!appendSupportEnabled) {
- throw new UnsupportedOperationException("Append Support not enabled");
- }
-
- LOG.debug("Opening file: {} for append", f);
-
- Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "append", absolutePath);
-
- String key = pathToKey(absolutePath);
- FileMetadata meta = null;
- try {
- meta = store.retrieveMetadata(key);
- } catch(Exception ex) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
-
- throw new FileNotFoundException(String.format("%s is not found", key));
- } else {
- throw ex;
- }
- }
-
- if (meta == null) {
- throw new FileNotFoundException(f.toString());
- }
-
- if (meta.isDirectory()) {
- throw new FileNotFoundException(f.toString()
- + " is a directory not a file.");
- }
-
- if (store.isPageBlobKey(key)) {
- throw new IOException("Append not supported for Page Blobs");
- }
-
- DataOutputStream appendStream = null;
-
- try {
- appendStream = store.retrieveAppendStream(key, bufferSize);
- } catch (Exception ex) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- } else {
- throw ex;
- }
- }
-
- return new FSDataOutputStream(appendStream, statistics);
- }
-
- @Override
- public FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
- return create(f, permission, overwrite, true,
- bufferSize, replication, blockSize, progress,
- (SelfRenewingLease) null);
- }
-
- /**
- * Get a self-renewing lease on the specified file.
- * @param path path whose lease to be renewed.
- * @return Lease
- * @throws AzureException when not being able to acquire a lease on the path
- */
- public SelfRenewingLease acquireLease(Path path) throws AzureException {
- String fullKey = pathToKey(makeAbsolute(path));
- return getStore().acquireLease(fullKey);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
-
- Path parent = f.getParent();
-
- // Get exclusive access to folder if this is a directory designated
- // for atomic rename. The primary use case of for HBase write-ahead
- // log file management.
- SelfRenewingLease lease = null;
- if (store.isAtomicRenameKey(pathToKey(f))) {
- try {
- lease = acquireLease(parent);
- } catch (AzureException e) {
-
- String errorCode = "";
- try {
- StorageException e2 = (StorageException) e.getCause();
- errorCode = e2.getErrorCode();
- } catch (Exception e3) {
- // do nothing if cast fails
- }
- if (errorCode.equals("BlobNotFound")) {
- throw new FileNotFoundException("Cannot create file " +
- f.getName() + " because parent folder does not exist.");
- }
-
- LOG.warn("Got unexpected exception trying to get lease on {} . {}",
- pathToKey(parent), e.getMessage());
- throw e;
- }
- }
-
- // See if the parent folder exists. If not, throw error.
- // The exists() check will push any pending rename operation forward,
- // if there is one, and return false.
- //
- // At this point, we have exclusive access to the source folder
- // via the lease, so we will not conflict with an active folder
- // rename operation.
- //
- // In the secure case, the call to exists will happen in the context
- // of the user that initiated the operation. In this case, we should
- // do the auth-check against ranger for the path.
- if (!exists(parent)) {
- try {
-
- // This'll let the keep-alive thread exit as soon as it wakes up.
- lease.free();
- } catch (Exception e) {
- LOG.warn("Unable to free lease because: {}", e.getMessage());
- }
- throw new FileNotFoundException("Cannot create file " +
- f.getName() + " because parent folder does not exist.");
- }
-
- // Create file inside folder.
- FSDataOutputStream out = null;
- try {
- out = create(f, permission, overwrite, false,
- bufferSize, replication, blockSize, progress, lease);
- } finally {
- // Release exclusive access to folder.
- try {
- if (lease != null) {
- lease.free();
- }
- } catch (Exception e) {
- NativeAzureFileSystemHelper.cleanup(LOG, out);
- String msg = "Unable to free lease on " + parent.toUri();
- LOG.error(msg);
- throw new IOException(msg, e);
- }
- }
- return out;
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
- EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
-
- // Check if file should be appended or overwritten. Assume that the file
- // is overwritten on if the CREATE and OVERWRITE create flags are set. Note
- // that any other combinations of create flags will result in an open new or
- // open with append.
- final EnumSet<CreateFlag> createflags =
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
- boolean overwrite = flags.containsAll(createflags);
-
- // Delegate the create non-recursive call.
- return this.createNonRecursive(f, permission, overwrite,
- bufferSize, replication, blockSize, progress);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
- return this.createNonRecursive(f, FsPermission.getFileDefault(),
- overwrite, bufferSize, replication, blockSize, progress);
- }
-
-
- /**
- * Create an Azure blob and return an output stream to use
- * to write data to it.
- *
- * @param f
- * @param permission
- * @param overwrite
- * @param createParent
- * @param bufferSize
- * @param replication
- * @param blockSize
- * @param progress
- * @param parentFolderLease Lease on parent folder (or null if
- * no lease).
- * @return
- * @throws IOException
- */
- private FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, boolean createParent, int bufferSize,
- short replication, long blockSize, Progressable progress,
- SelfRenewingLease parentFolderLease)
- throws FileAlreadyExistsException, IOException {
-
- LOG.debug("Creating file: {}", f.toString());
-
- if (containsColon(f)) {
- throw new IOException("Cannot create file " + f
- + " through WASB that has colons in the name");
- }
-
- Path absolutePath = makeAbsolute(f);
- Path ancestor = getAncestor(absolutePath);
-
- performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "create", absolutePath);
-
- return createInternal(f, permission, overwrite, parentFolderLease);
- }
-
-
- /**
- * This is the version of the create call that is meant for internal usage.
- * This version is not public facing and does not perform authorization checks.
- * It is used by the public facing create call and by FolderRenamePending to
- * create the internal -RenamePending.json file.
- * @param f the path to a file to be created.
- * @param permission for the newly created file.
- * @param overwrite specifies if the file should be overwritten.
- * @param parentFolderLease lease on the parent folder.
- * @return the output stream used to write data into the newly created file .
- * @throws IOException if an IO error occurs while attempting to delete the
- * path.
- *
- */
- protected FSDataOutputStream createInternal(Path f, FsPermission permission,
- boolean overwrite,
- SelfRenewingLease parentFolderLease)
- throws FileAlreadyExistsException, IOException {
-
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
-
- FileMetadata existingMetadata = store.retrieveMetadata(key);
- if (existingMetadata != null) {
- if (existingMetadata.isDirectory()) {
- throw new FileAlreadyExistsException("Cannot create file " + f
- + "; already exists as a directory.");
- }
- if (!overwrite) {
- throw new FileAlreadyExistsException("File already exists:" + f);
- }
- else {
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "create", absolutePath);
- }
- }
-
- Path parentFolder = absolutePath.getParent();
- if (parentFolder != null && parentFolder.getParent() != null) { // skip root
- // Update the parent folder last modified time if the parent folder
- // already exists.
- String parentKey = pathToKey(parentFolder);
- FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
- if (parentMetadata != null && parentMetadata.isDirectory() &&
- parentMetadata.getBlobMaterialization() == BlobMaterialization.Explicit) {
- if (parentFolderLease != null) {
- store.updateFolderLastModifiedTime(parentKey, parentFolderLease);
- } else {
- updateParentFolderLastModifiedTime(key);
- }
- } else {
- // Make sure that the parent folder exists.
- // Create it using inherited permissions from the first existing directory going up the path
- Path firstExisting = parentFolder.getParent();
- FileMetadata metadata = store.retrieveMetadata(pathToKey(firstExisting));
- while(metadata == null) {
- // Guaranteed to terminate properly because we will eventually hit root, which will return non-null metadata
- firstExisting = firstExisting.getParent();
- metadata = store.retrieveMetadata(pathToKey(firstExisting));
- }
- mkdirs(parentFolder, metadata.getPermission(), true);
- }
- }
-
- // Mask the permission first (with the default permission mask as well).
- FsPermission masked = applyUMask(permission, UMaskApplyMode.NewFile);
- PermissionStatus permissionStatus = createPermissionStatus(masked);
-
- OutputStream bufOutStream;
- if (store.isPageBlobKey(key)) {
- // Store page blobs directly in-place without renames.
- bufOutStream = store.storefile(key, permissionStatus, key);
- } else {
- // This is a block blob, so open the output blob stream based on the
- // encoded key.
- //
- String keyEncoded = encodeKey(key);
-
-
- // First create a blob at the real key, pointing back to the temporary file
- // This accomplishes a few things:
- // 1. Makes sure we can create a file there.
- // 2. Makes it visible to other concurrent threads/processes/nodes what
- // we're
- // doing.
- // 3. Makes it easier to restore/cleanup data in the event of us crashing.
- store.storeEmptyLinkFile(key, keyEncoded, permissionStatus);
-
- // The key is encoded to point to a common container at the storage server.
- // This reduces the number of splits on the server side when load balancing.
- // Ingress to Azure storage can take advantage of earlier splits. We remove
- // the root path to the key and prefix a random GUID to the tail (or leaf
- // filename) of the key. Keys are thus broadly and randomly distributed over
- // a single container to ease load balancing on the storage server. When the
- // blob is committed it is renamed to its earlier key. Uncommitted blocks
- // are not cleaned up and we leave it to Azure storage to garbage collect
- // these
- // blocks.
- bufOutStream = new NativeAzureFsOutputStream(store.storefile(
- keyEncoded, permissionStatus, key), key, keyEncoded);
- }
- // Construct the data output stream from the buffered output stream.
- FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics);
-
-
- // Increment the counter
- instrumentation.fileCreated();
-
- // Return data output stream to caller.
- return fsOut;
- }
-
- @Override
- @Deprecated
- public boolean delete(Path path) throws IOException {
- return delete(path, true);
- }
-
- @Override
- public boolean delete(Path f, boolean recursive) throws IOException {
- return delete(f, recursive, false);
- }
-
- /**
- * Delete file or folder with authorization checks. Most of the code
- * is duplicate of the actual delete implementation and will be merged
- * once the performance and funcional aspects are guaranteed not to
- * regress existing delete semantics.
- */
- private boolean deleteWithAuthEnabled(Path f, boolean recursive,
- boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
-
- LOG.debug("Deleting file: {}", f);
-
- Path absolutePath = makeAbsolute(f);
- Path parentPath = absolutePath.getParent();
-
- // If delete is issued for 'root', parentPath will be null
- // In that case, we perform auth check for root itself before
- // proceeding for deleting contents under root.
- if (parentPath != null) {
- performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- } else {
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- }
-
- String key = pathToKey(absolutePath);
-
- // Capture the metadata for the path.
- FileMetadata metaFile = null;
- try {
- metaFile = store.retrieveMetadata(key);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
-
- return false;
- }
- throw e;
- }
-
- if (null == metaFile) {
- // The path to be deleted does not exist.
- return false;
- }
-
- FileMetadata parentMetadata = null;
- String parentKey = null;
- if (parentPath != null) {
- parentKey = pathToKey(parentPath);
-
- try {
- parentMetadata = store.retrieveMetadata(parentKey);
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
- if (innerException instanceof StorageException) {
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (isFileNotFoundException((StorageException) innerException)) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
- }
- throw e;
- }
-
- // Same case as unable to retrieve metadata
- if (parentMetadata == null) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
-
- if (!parentMetadata.isDirectory()) {
- // Invalid state: the parent path is actually a file. Throw.
- throw new AzureException("File " + f + " has a parent directory "
- + parentPath + " which is also a file. Can't resolve.");
- }
- }
-
- // The path exists, determine if it is a folder containing objects,
- // an empty folder, or a simple file and take the appropriate actions.
- if (!metaFile.isDirectory()) {
- // The path specifies a file. We need to check the parent path
- // to make sure it's a proper materialized directory before we
- // delete the file. Otherwise we may get into a situation where
- // the file we were deleting was the last one in an implicit directory
- // (e.g. the blob store only contains the blob a/b and there's no
- // corresponding directory blob a) and that would implicitly delete
- // the directory as well, which is not correct.
-
- if (parentPath != null && parentPath.getParent() != null) {// Not root
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the file {}. Creating the directory blob for"
- + " it in {}.", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- } else {
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- // check if the file can be deleted based on sticky bit check
- // This check will be performed only when authorization is enabled
- if (isStickyBitCheckViolated(metaFile, parentMetadata)) {
- throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
- + "File %s cannot be deleted.", parentPath, f));
- }
-
- try {
- if (store.delete(key)) {
- instrumentation.fileDeleted();
- } else {
- return false;
- }
- } catch(IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
-
- throw e;
- }
- } else {
- // The path specifies a folder. Recursively delete all entries under the
- // folder.
- LOG.debug("Directory Delete encountered: {}", f);
- if (parentPath != null && parentPath.getParent() != null) {
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the directory {}. Creating the directory blob for"
- + " it in {}. ", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- }
- }
-
- // check if the folder can be deleted based on sticky bit check on parent
- // This check will be performed only when authorization is enabled.
- if (!metaFile.getKey().equals("/")
- && isStickyBitCheckViolated(metaFile, parentMetadata)) {
-
- throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
- + "File %s cannot be deleted.", parentPath, f));
- }
-
- // Iterate through folder contents and get the list of files
- // and folders that can be deleted. We might encounter IOException
- // while listing blobs. In such cases, we return false.
- ArrayList<FileMetadata> fileMetadataList = new ArrayList<>();
- boolean isPartialDelete = false;
-
- // Start time for list operation
- long start = Time.monotonicNow();
-
- try {
- // Get list of files/folders that can be deleted
- // based on authorization checks and stickybit checks
- isPartialDelete = getFolderContentsToDelete(metaFile, fileMetadataList);
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
- throw e;
- }
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for delete operation: {} ms",
- fileMetadataList.size(), (end - start));
-
- // Here contents holds the list of metadata of the files and folders that can be deleted
- // under the path that is requested for delete(excluding itself).
- final FileMetadata[] contents = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
-
- if (contents.length > 0 && !recursive) {
- // The folder is non-empty and recursive delete was not specified.
- // Throw an exception indicating that a non-recursive delete was
- // specified for a non-empty folder.
- throw new IOException("Non-recursive delete of non-empty directory "
- + f);
- }
-
- // Delete all files / folders in current directory stored as list in 'contents'.
- AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
- @Override
- public boolean execute(FileMetadata file) throws IOException{
- if (!deleteFile(file.getKey(), file.isDirectory())) {
- LOG.warn("Attempt to delete non-existent {} {}",
- file.isDirectory() ? "directory" : "file",
- file.getKey());
- }
- return true;
- }
- };
-
- AzureFileSystemThreadPoolExecutor executor = getThreadPoolExecutor(this.deleteThreadCount,
- "AzureBlobDeleteThread", "Delete", key, AZURE_DELETE_THREADS);
-
- if (!executor.executeParallel(contents, task)) {
- LOG.error("Failed to delete files / subfolders in blob {}", key);
- return false;
- }
-
- if (metaFile.getKey().equals("/")) {
- LOG.error("Cannot delete root directory {}", f);
- return false;
- }
-
- // Delete the current directory if all underlying contents are deleted
- if (isPartialDelete || (store.retrieveMetadata(metaFile.getKey()) != null
- && !deleteFile(metaFile.getKey(), metaFile.isDirectory()))) {
- LOG.error("Failed delete directory : {}", f);
- return false;
- }
-
- // Update parent directory last modified time
- Path parent = absolutePath.getParent();
- if (parent != null && parent.getParent() != null) { // not root
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- // File or directory was successfully deleted.
- LOG.debug("Delete Successful for : {}", f);
- return true;
- }
-
- private boolean deleteWithoutAuth(Path f, boolean recursive,
- boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
-
- LOG.debug("Deleting file: {}", f);
-
- Path absolutePath = makeAbsolute(f);
- Path parentPath = absolutePath.getParent();
-
- String key = pathToKey(absolutePath);
-
- // Capture the metadata for the path.
- //
- FileMetadata metaFile = null;
- try {
- metaFile = store.retrieveMetadata(key);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
-
- return false;
- }
- throw e;
- }
-
- if (null == metaFile) {
- // The path to be deleted does not exist.
- return false;
- }
-
- // The path exists, determine if it is a folder containing objects,
- // an empty folder, or a simple file and take the appropriate actions.
- if (!metaFile.isDirectory()) {
- // The path specifies a file. We need to check the parent path
- // to make sure it's a proper materialized directory before we
- // delete the file. Otherwise we may get into a situation where
- // the file we were deleting was the last one in an implicit directory
- // (e.g. the blob store only contains the blob a/b and there's no
- // corresponding directory blob a) and that would implicitly delete
- // the directory as well, which is not correct.
-
- if (parentPath.getParent() != null) {// Not root
- String parentKey = pathToKey(parentPath);
-
- FileMetadata parentMetadata = null;
- try {
- parentMetadata = store.retrieveMetadata(parentKey);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (isFileNotFoundException((StorageException) innerException)) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
- }
- throw e;
- }
-
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (parentMetadata == null) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
-
- if (!parentMetadata.isDirectory()) {
- // Invalid state: the parent path is actually a file. Throw.
- throw new AzureException("File " + f + " has a parent directory "
- + parentPath + " which is also a file. Can't resolve.");
- }
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the file {}. Creating the directory blob for"
- + " it in {}.", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- } else {
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- try {
- if (store.delete(key)) {
- instrumentation.fileDeleted();
- } else {
- return false;
- }
- } catch(IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
-
- throw e;
- }
- } else {
- // The path specifies a folder. Recursively delete all entries under the
- // folder.
- LOG.debug("Directory Delete encountered: {}", f);
- if (parentPath.getParent() != null) {
- String parentKey = pathToKey(parentPath);
- FileMetadata parentMetadata = null;
-
- try {
- parentMetadata = store.retrieveMetadata(parentKey);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (isFileNotFoundException((StorageException) innerException)) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
- }
- throw e;
- }
-
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (parentMetadata == null) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the directory {}. Creating the directory blob for"
- + " it in {}. ", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- }
- }
-
- // Start time for list operation
- long start = Time.monotonicNow();
- final FileMetadata[] contents;
-
- // List all the files in the folder with AZURE_UNBOUNDED_DEPTH depth.
- try {
- contents = store.list(key, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH);
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
-
- throw e;
- }
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for delete operation: {} ms", contents.length, (end - start));
-
- if (contents.length > 0) {
- if (!recursive) {
- // The folder is non-empty and recursive delete was not specified.
- // Throw an exception indicating that a non-recursive delete was
- // specified for a non-empty folder.
- throw new IOException("Non-recursive delete of non-empty directory "+ f);
- }
- }
-
- // Delete all files / folders in current directory stored as list in 'contents'.
- AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
- @Override
- public boolean execute(FileMetadata file) throws IOException{
- if (!deleteFile(file.getKey(), file.isDirectory())) {
- LOG.warn("Attempt to delete non-existent {} {}",
- file.isDirectory() ? "directory" : "file",
- file.getKey());
- }
- return true;
- }
- };
-
- AzureFileSystemThreadPoolExecutor executor = getThreadPoolExecutor(this.deleteThreadCount,
- "AzureBlobDeleteThread", "Delete", key, AZURE_DELETE_THREADS);
-
- if (!executor.executeParallel(contents, task)) {
- LOG.error("Failed to delete files / subfolders in blob {}", key);
- return false;
- }
-
- // Delete the current directory
- if (store.retrieveMetadata(metaFile.getKey()) != null
- && !deleteFile(metaFile.getKey(), metaFile.isDirectory())) {
- LOG.error("Failed delete directory : {}", f);
- return false;
- }
-
- // Update parent directory last modified time
- Path parent = absolutePath.getParent();
- if (parent != null && parent.getParent() != null) { // not root
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- // File or directory was successfully deleted.
- LOG.debug("Delete Successful for : {}", f);
- return true;
- }
-
- /**
- * Delete the specified file or folder. The parameter
- * skipParentFolderLastModifiedTimeUpdate
- * is used in the case of atomic folder rename redo. In that case, there is
- * a lease on the parent folder, so (without reworking the code) modifying
- * the parent folder update time will fail because of a conflict with the
- * lease. Since we are going to delete the folder soon anyway so accurate
- * modified time is not necessary, it's easier to just skip
- * the modified time update.
- *
- * @param f file path to be deleted.
- * @param recursive specify deleting recursively or not.
- * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last
- * modified time.
- * @return true if and only if the file is deleted
- * @throws IOException Thrown when fail to delete file or directory.
- */
- public boolean delete(Path f, boolean recursive,
- boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
-
- if (this.azureAuthorization) {
- return deleteWithAuthEnabled(f, recursive,
- skipParentFolderLastModifiedTimeUpdate);
- } else {
... 33383 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org