You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2026/02/24 18:25:36 UTC
(hadoop) branch branch-3.5 updated (e4cc43c31e3 -> 502dfaf5910)
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a change to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from e4cc43c31e3 HADOOP-19779: Follow-up fix in create-release script to parse JDK version from root pom.xml instead of hadoop-project/pom.xml
new 54d1dd00874 HADOOP-19810. Replace FastByteComparisons with JDK Arrays.compareUnsigned (#8239) Contributed by Cheng Pan.
new 502dfaf5910 HADOOP-19778. [ABFS] Removing Deprecated Wasb Code
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/hadoop/io/FastByteComparisons.java | 245 --
.../org/apache/hadoop/io/WritableComparator.java | 3 +-
hadoop-tools/hadoop-azure/pom.xml | 245 +-
.../org/apache/hadoop/fs/azure/AzureException.java | 44 -
.../azure/AzureFileSystemThreadPoolExecutor.java | 347 --
.../hadoop/fs/azure/AzureFileSystemThreadTask.java | 29 -
.../apache/hadoop/fs/azure/AzureLinkedStack.java | 217 --
.../fs/azure/AzureNativeFileSystemStore.java | 3105 ---------------
.../hadoop/fs/azure/BlobMaterialization.java | 42 -
.../hadoop/fs/azure/BlobOperationDescriptor.java | 222 --
.../hadoop/fs/azure/BlockBlobAppendStream.java | 1141 ------
.../hadoop/fs/azure/BlockBlobInputStream.java | 482 ---
.../apache/hadoop/fs/azure/CachingAuthorizer.java | 232 --
.../hadoop/fs/azure/ClientThrottlingAnalyzer.java | 284 --
.../hadoop/fs/azure/ClientThrottlingIntercept.java | 221 --
.../org/apache/hadoop/fs/azure/FileMetadata.java | 125 -
.../org/apache/hadoop/fs/azure/KeyProvider.java | 44 -
.../hadoop/fs/azure/KeyProviderException.java | 42 -
.../hadoop/fs/azure/LocalSASKeyGeneratorImpl.java | 295 --
.../hadoop/fs/azure/NativeAzureFileSystem.java | 4058 +-------------------
.../fs/azure/NativeAzureFileSystemHelper.java | 155 -
.../hadoop/fs/azure/NativeFileSystemStore.java | 148 -
.../hadoop/fs/azure/PageBlobFormatHelpers.java | 58 -
.../hadoop/fs/azure/PageBlobInputStream.java | 506 ---
.../hadoop/fs/azure/PageBlobOutputStream.java | 607 ---
.../hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java | 337 --
.../hadoop/fs/azure/RemoteWasbAuthorizerImpl.java | 250 --
.../hadoop/fs/azure/SASKeyGenerationException.java | 40 -
.../hadoop/fs/azure/SASKeyGeneratorImpl.java | 61 -
.../hadoop/fs/azure/SASKeyGeneratorInterface.java | 66 -
.../hadoop/fs/azure/SecureModeException.java | 40 -
.../fs/azure/SecureStorageInterfaceImpl.java | 606 ---
.../fs/azure/SecureWasbRemoteCallHelper.java | 230 --
.../apache/hadoop/fs/azure/SelfRenewingLease.java | 213 -
.../hadoop/fs/azure/SelfThrottlingIntercept.java | 195 -
.../hadoop/fs/azure/SendRequestIntercept.java | 92 -
.../fs/azure/ShellDecryptionKeyProvider.java | 62 -
.../apache/hadoop/fs/azure/SimpleKeyProvider.java | 61 -
.../apache/hadoop/fs/azure/StorageInterface.java | 793 ----
.../hadoop/fs/azure/StorageInterfaceImpl.java | 522 ---
.../hadoop/fs/azure/SyncableDataOutputStream.java | 106 -
.../main/java/org/apache/hadoop/fs/azure/Wasb.java | 47 -
.../fs/azure/WasbAuthorizationException.java | 40 -
.../fs/azure/WasbAuthorizationOperations.java | 42 -
.../hadoop/fs/azure/WasbAuthorizerInterface.java | 53 -
.../java/org/apache/hadoop/fs/azure/WasbFsck.java | 200 -
.../hadoop/fs/azure/WasbRemoteCallException.java | 41 -
.../hadoop/fs/azure/WasbRemoteCallHelper.java | 297 --
.../java/org/apache/hadoop/fs/azure/Wasbs.java | 47 -
.../metrics/AzureFileSystemInstrumentation.java | 396 --
.../metrics/AzureFileSystemMetricsSystem.java | 69 -
.../fs/azure/metrics/BandwidthGaugeUpdater.java | 286 --
.../fs/azure/metrics/ErrorMetricUpdater.java | 82 -
.../metrics/ResponseReceivedMetricUpdater.java | 142 -
.../fs/azure/metrics/RollingWindowAverage.java | 103 -
.../apache/hadoop/fs/azure/metrics/package.html | 28 -
.../org/apache/hadoop/fs/azure}/package-info.java | 5 +-
.../java/org/apache/hadoop/fs/azure/package.html | 31 -
.../apache/hadoop/fs/azure/security/Constants.java | 50 -
.../apache/hadoop/fs/azure/security/JsonUtils.java | 52 -
.../security/RemoteWasbDelegationTokenManager.java | 169 -
.../hadoop/fs/azure/security/SpnegoToken.java | 49 -
.../hadoop/fs/azure/security/TokenUtils.java | 60 -
.../security/WasbDelegationTokenIdentifier.java | 48 -
.../azure/security/WasbDelegationTokenManager.java | 54 -
.../hadoop/fs/azure/security/WasbTokenRenewer.java | 91 -
.../hadoop/fs/azure/security/package-info.java | 22 -
...rg.apache.hadoop.security.token.TokenIdentifier | 1 -
.../org.apache.hadoop.security.token.TokenRenewer | 1 -
.../src/site/markdown/deprecated_wasb.md | 564 ---
.../hadoop-azure/src/site/markdown/fns_blob.md | 12 +-
.../hadoop-azure/src/site/markdown/index.md | 6 +-
.../src/site/markdown/testing_azure.md | 174 +-
.../hadoop-azure/src/site/markdown/wasb.md | 30 +-
.../src/site/markdown/wasbToAbfsMigration.md | 8 +-
.../hadoop/fs/azure/AbstractWasbTestBase.java | 175 -
.../fs/azure/AbstractWasbTestWithTimeout.java | 78 -
.../fs/azure/AzureBlobStorageTestAccount.java | 947 -----
.../fs/azure/ExceptionHandlingTestHelper.java | 67 -
.../fs/azure/ITestAzureConcurrentOutOfBandIo.java | 180 -
...stAzureConcurrentOutOfBandIoWithSecureMode.java | 33 -
.../azure/ITestAzureFileSystemErrorConditions.java | 242 --
.../hadoop/fs/azure/ITestBlobDataValidation.java | 242 --
.../fs/azure/ITestBlobTypeSpeedDifference.java | 163 -
.../hadoop/fs/azure/ITestBlockBlobInputStream.java | 927 -----
.../hadoop/fs/azure/ITestContainerChecks.java | 190 -
.../ITestFileSystemOperationExceptionHandling.java | 319 --
.../ITestFileSystemOperationExceptionMessage.java | 80 -
...emOperationsExceptionHandlingMultiThreaded.java | 381 --
.../ITestFileSystemOperationsWithThreads.java | 812 ----
.../hadoop/fs/azure/ITestListPerformance.java | 198 -
...ITestNativeAzureFSAuthWithBlobSpecificKeys.java | 40 -
.../ITestNativeAzureFSAuthorizationCaching.java | 53 -
.../fs/azure/ITestNativeAzureFSPageBlobLive.java | 43 -
.../fs/azure/ITestNativeAzureFileSystemAppend.java | 354 --
...stNativeAzureFileSystemAtomicRenameDirList.java | 55 -
.../ITestNativeAzureFileSystemClientLogging.java | 136 -
.../ITestNativeAzureFileSystemConcurrencyLive.java | 241 --
...ITestNativeAzureFileSystemContractEmulator.java | 67 -
.../ITestNativeAzureFileSystemContractLive.java | 107 -
...tNativeAzureFileSystemContractPageBlobLive.java | 115 -
.../fs/azure/ITestNativeAzureFileSystemLive.java | 309 --
.../fs/azure/ITestNativeFileSystemStatistics.java | 98 -
.../ITestOutOfBandAzureBlobOperationsLive.java | 185 -
.../fs/azure/ITestOutputStreamSemantics.java | 424 --
.../hadoop/fs/azure/ITestPageBlobInputStream.java | 523 ---
.../hadoop/fs/azure/ITestPageBlobOutputStream.java | 83 -
.../azure/ITestReadAndSeekPageBlobAfterWrite.java | 342 --
.../hadoop/fs/azure/ITestWasbRemoteCallHelper.java | 517 ---
.../fs/azure/ITestWasbUriAndConfiguration.java | 664 ----
.../hadoop/fs/azure/InMemoryBlockBlobStore.java | 197 -
.../hadoop/fs/azure/MockStorageInterface.java | 661 ----
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 215 --
.../fs/azure/NativeAzureFileSystemBaseTest.java | 1731 ---------
.../apache/hadoop/fs/azure/TestBlobMetadata.java | 269 --
.../fs/azure/TestBlobOperationDescriptor.java | 303 --
.../fs/azure/TestClientThrottlingAnalyzer.java | 167 -
.../fs/azure/TestKeyPageBlobDirectories.java | 169 -
.../TestNativeAzureFileSystemAuthorization.java | 2113 ----------
.../TestNativeAzureFileSystemBlockCompaction.java | 265 --
.../TestNativeAzureFileSystemConcurrency.java | 181 -
.../TestNativeAzureFileSystemContractMocked.java | 66 -
.../TestNativeAzureFileSystemFileNameCheck.java | 131 -
.../fs/azure/TestNativeAzureFileSystemMocked.java | 67 -
.../TestNativeAzureFileSystemOperationsMocked.java | 89 -
.../TestNativeAzureFileSystemUploadLogic.java | 181 -
.../fs/azure/TestOutOfBandAzureBlobOperations.java | 170 -
.../fs/azure/TestShellDecryptionKeyProvider.java | 81 -
.../fs/azure/TestSyncableDataOutputStream.java | 68 -
.../org/apache/hadoop/fs/azure/TestWasbFsck.java | 134 -
.../hadoop/fs/azure/TestWasbInitFailure.java | 71 +
.../contract/ITestAzureNativeContractAppend.java | 41 -
.../contract/ITestAzureNativeContractCreate.java | 34 -
.../contract/ITestAzureNativeContractDelete.java | 33 -
.../contract/ITestAzureNativeContractDistCp.java | 49 -
.../ITestAzureNativeContractGetFileStatus.java | 35 -
.../contract/ITestAzureNativeContractMkdir.java | 33 -
.../contract/ITestAzureNativeContractOpen.java | 34 -
.../contract/ITestAzureNativeContractRename.java | 34 -
.../contract/ITestAzureNativeContractSeek.java | 34 -
.../contract/NativeAzureFileSystemContract.java | 49 -
.../azure/integration/AbstractAzureScaleTest.java | 71 -
.../fs/azure/integration/AzureTestConstants.java | 191 -
.../fs/azure/integration/AzureTestUtils.java | 557 ---
.../azure/integration/CleanupTestContainers.java | 87 -
.../fs/azure/integration/ITestAzureHugeFiles.java | 459 ---
.../apache/hadoop/fs/azure/integration/Sizes.java | 43 -
.../fs/azure/metrics/AzureMetricsTestUtil.java | 83 -
.../ITestAzureFileSystemInstrumentation.java | 546 ---
.../azure/metrics/TestBandwidthGaugeUpdater.java | 78 -
.../TestNativeAzureFileSystemMetricsSystem.java | 100 -
.../fs/azure/metrics/TestRollingWindowAverage.java | 42 -
.../{azurebfs/oauth2 => azure}/package-info.java | 5 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 45 -
.../hadoop/fs/azurebfs/AbstractAbfsScaleTest.java | 12 +-
.../hadoop/fs/azurebfs/ITestAbfsHugeFiles.java | 10 +-
.../ITestAzureBlobFileSystemRandomRead.java | 103 -
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2044 ----------
.../azurebfs/commit/AbstractAbfsClusterITest.java | 6 +-
.../commit/ITestAbfsLoadManifestsStage.java | 5 +-
.../azurebfs/constants/TestConfigurationKeys.java | 3 +
.../ITestAbfsFileSystemContractDistCp.java | 6 +-
.../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 194 +-
.../hadoop-azure/src/test/resources/wasb.xml | 169 -
164 files changed, 401 insertions(+), 41424 deletions(-)
delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadTask.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGenerationException.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorInterface.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureModeException.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ShellDecryptionKeyProvider.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallException.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemInstrumentation.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/BandwidthGaugeUpdater.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/RollingWindowAverage.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/package.html
copy hadoop-tools/hadoop-azure/src/{test/java/org/apache/hadoop/fs/azurebfs/oauth2 => main/java/org/apache/hadoop/fs/azure}/package-info.java (95%)
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/RemoteWasbDelegationTokenManager.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SpnegoToken.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/TokenUtils.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenManager.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java
delete mode 100644 hadoop-tools/hadoop-azure/src/site/markdown/deprecated_wasb.md
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ExceptionHandlingTestHelper.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobTypeSpeedDifference.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlockBlobInputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionHandling.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsWithThreads.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSPageBlobLive.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAtomicRenameDirList.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemClientLogging.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemConcurrencyLive.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemContractEmulator.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemContractLive.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemContractPageBlobLive.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeFileSystemStatistics.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutOfBandAzureBlobOperationsLive.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutputStreamSemantics.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestPageBlobInputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestPageBlobOutputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestReadAndSeekPageBlobAfterWrite.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbRemoteCallHelper.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobMetadata.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobOperationDescriptor.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestClientThrottlingAnalyzer.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemConcurrency.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemContractMocked.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemFileNameCheck.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemMocked.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemOperationsMocked.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemUploadLogic.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperations.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestShellDecryptionKeyProvider.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestSyncableDataOutputStream.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbFsck.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbInitFailure.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/CleanupTestContainers.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/ITestAzureHugeFiles.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/Sizes.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/AzureMetricsTestUtil.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java
copy hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/{azurebfs/oauth2 => azure}/package-info.java (95%)
delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
delete mode 100644 hadoop-tools/hadoop-azure/src/test/resources/wasb.xml
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 02/02: HADOOP-19778. [ABFS] Removing Deprecated Wasb Code
Posted by cn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 502dfaf591084ee94822eafc88001ddd2581f43b
Author: Anuj Modi <an...@apache.org>
AuthorDate: Tue Feb 24 17:14:09 2026 +0000
HADOOP-19778. [ABFS] Removing Deprecated Wasb Code
Closes #8207
Signed-off-by: Chris Nauroth <cn...@apache.org>
Reviewed-by: Anmol Asrani <an...@gmail.com>
Reviewed-by: manika137 <80...@users.noreply.github.com>
---
hadoop-tools/hadoop-azure/pom.xml | 245 +-
.../org/apache/hadoop/fs/azure/AzureException.java | 44 -
.../azure/AzureFileSystemThreadPoolExecutor.java | 347 --
.../apache/hadoop/fs/azure/AzureLinkedStack.java | 217 --
.../fs/azure/AzureNativeFileSystemStore.java | 3105 ---------------
.../hadoop/fs/azure/BlobMaterialization.java | 42 -
.../hadoop/fs/azure/BlobOperationDescriptor.java | 222 --
.../hadoop/fs/azure/BlockBlobAppendStream.java | 1141 ------
.../hadoop/fs/azure/BlockBlobInputStream.java | 482 ---
.../apache/hadoop/fs/azure/CachingAuthorizer.java | 232 --
.../hadoop/fs/azure/ClientThrottlingAnalyzer.java | 284 --
.../hadoop/fs/azure/ClientThrottlingIntercept.java | 221 --
.../org/apache/hadoop/fs/azure/FileMetadata.java | 125 -
.../org/apache/hadoop/fs/azure/KeyProvider.java | 44 -
.../hadoop/fs/azure/KeyProviderException.java | 42 -
.../hadoop/fs/azure/LocalSASKeyGeneratorImpl.java | 295 --
.../hadoop/fs/azure/NativeAzureFileSystem.java | 4058 +-------------------
.../fs/azure/NativeAzureFileSystemHelper.java | 155 -
.../hadoop/fs/azure/NativeFileSystemStore.java | 148 -
.../hadoop/fs/azure/PageBlobFormatHelpers.java | 58 -
.../hadoop/fs/azure/PageBlobInputStream.java | 506 ---
.../hadoop/fs/azure/PageBlobOutputStream.java | 607 ---
.../hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java | 337 --
.../hadoop/fs/azure/RemoteWasbAuthorizerImpl.java | 250 --
.../hadoop/fs/azure/SASKeyGenerationException.java | 40 -
.../hadoop/fs/azure/SASKeyGeneratorImpl.java | 61 -
.../hadoop/fs/azure/SASKeyGeneratorInterface.java | 66 -
.../hadoop/fs/azure/SecureModeException.java | 40 -
.../fs/azure/SecureStorageInterfaceImpl.java | 606 ---
.../fs/azure/SecureWasbRemoteCallHelper.java | 230 --
.../apache/hadoop/fs/azure/SelfRenewingLease.java | 213 -
.../hadoop/fs/azure/SelfThrottlingIntercept.java | 195 -
.../hadoop/fs/azure/SendRequestIntercept.java | 92 -
.../fs/azure/ShellDecryptionKeyProvider.java | 62 -
.../apache/hadoop/fs/azure/SimpleKeyProvider.java | 61 -
.../apache/hadoop/fs/azure/StorageInterface.java | 793 ----
.../hadoop/fs/azure/StorageInterfaceImpl.java | 522 ---
.../hadoop/fs/azure/SyncableDataOutputStream.java | 106 -
.../main/java/org/apache/hadoop/fs/azure/Wasb.java | 47 -
.../fs/azure/WasbAuthorizationException.java | 40 -
.../fs/azure/WasbAuthorizationOperations.java | 42 -
.../hadoop/fs/azure/WasbAuthorizerInterface.java | 53 -
.../java/org/apache/hadoop/fs/azure/WasbFsck.java | 200 -
.../hadoop/fs/azure/WasbRemoteCallException.java | 41 -
.../hadoop/fs/azure/WasbRemoteCallHelper.java | 297 --
.../java/org/apache/hadoop/fs/azure/Wasbs.java | 47 -
.../metrics/AzureFileSystemInstrumentation.java | 396 --
.../metrics/AzureFileSystemMetricsSystem.java | 69 -
.../fs/azure/metrics/BandwidthGaugeUpdater.java | 286 --
.../fs/azure/metrics/ErrorMetricUpdater.java | 82 -
.../metrics/ResponseReceivedMetricUpdater.java | 142 -
.../fs/azure/metrics/RollingWindowAverage.java | 103 -
.../apache/hadoop/fs/azure/metrics/package.html | 28 -
.../fs/azure/{security => }/package-info.java | 9 +-
.../java/org/apache/hadoop/fs/azure/package.html | 31 -
.../apache/hadoop/fs/azure/security/Constants.java | 50 -
.../apache/hadoop/fs/azure/security/JsonUtils.java | 52 -
.../security/RemoteWasbDelegationTokenManager.java | 169 -
.../hadoop/fs/azure/security/SpnegoToken.java | 49 -
.../hadoop/fs/azure/security/TokenUtils.java | 60 -
.../security/WasbDelegationTokenIdentifier.java | 48 -
.../azure/security/WasbDelegationTokenManager.java | 54 -
.../hadoop/fs/azure/security/WasbTokenRenewer.java | 91 -
...rg.apache.hadoop.security.token.TokenIdentifier | 1 -
.../org.apache.hadoop.security.token.TokenRenewer | 1 -
.../src/site/markdown/deprecated_wasb.md | 564 ---
.../hadoop-azure/src/site/markdown/fns_blob.md | 12 +-
.../hadoop-azure/src/site/markdown/index.md | 6 +-
.../src/site/markdown/testing_azure.md | 174 +-
.../hadoop-azure/src/site/markdown/wasb.md | 30 +-
.../src/site/markdown/wasbToAbfsMigration.md | 8 +-
.../hadoop/fs/azure/AbstractWasbTestBase.java | 175 -
.../fs/azure/AbstractWasbTestWithTimeout.java | 78 -
.../fs/azure/AzureBlobStorageTestAccount.java | 947 -----
.../fs/azure/ExceptionHandlingTestHelper.java | 67 -
.../fs/azure/ITestAzureConcurrentOutOfBandIo.java | 180 -
...stAzureConcurrentOutOfBandIoWithSecureMode.java | 33 -
.../azure/ITestAzureFileSystemErrorConditions.java | 242 --
.../hadoop/fs/azure/ITestBlobDataValidation.java | 242 --
.../fs/azure/ITestBlobTypeSpeedDifference.java | 163 -
.../hadoop/fs/azure/ITestBlockBlobInputStream.java | 927 -----
.../hadoop/fs/azure/ITestContainerChecks.java | 190 -
.../ITestFileSystemOperationExceptionHandling.java | 319 --
.../ITestFileSystemOperationExceptionMessage.java | 80 -
...emOperationsExceptionHandlingMultiThreaded.java | 381 --
.../ITestFileSystemOperationsWithThreads.java | 812 ----
.../hadoop/fs/azure/ITestListPerformance.java | 198 -
...ITestNativeAzureFSAuthWithBlobSpecificKeys.java | 40 -
.../ITestNativeAzureFSAuthorizationCaching.java | 53 -
.../fs/azure/ITestNativeAzureFSPageBlobLive.java | 43 -
.../fs/azure/ITestNativeAzureFileSystemAppend.java | 354 --
...stNativeAzureFileSystemAtomicRenameDirList.java | 55 -
.../ITestNativeAzureFileSystemClientLogging.java | 136 -
.../ITestNativeAzureFileSystemConcurrencyLive.java | 241 --
...ITestNativeAzureFileSystemContractEmulator.java | 67 -
.../ITestNativeAzureFileSystemContractLive.java | 107 -
...tNativeAzureFileSystemContractPageBlobLive.java | 115 -
.../fs/azure/ITestNativeAzureFileSystemLive.java | 309 --
.../fs/azure/ITestNativeFileSystemStatistics.java | 98 -
.../ITestOutOfBandAzureBlobOperationsLive.java | 185 -
.../fs/azure/ITestOutputStreamSemantics.java | 424 --
.../hadoop/fs/azure/ITestPageBlobInputStream.java | 523 ---
.../hadoop/fs/azure/ITestPageBlobOutputStream.java | 83 -
.../azure/ITestReadAndSeekPageBlobAfterWrite.java | 342 --
.../hadoop/fs/azure/ITestWasbRemoteCallHelper.java | 517 ---
.../fs/azure/ITestWasbUriAndConfiguration.java | 664 ----
.../hadoop/fs/azure/InMemoryBlockBlobStore.java | 197 -
.../hadoop/fs/azure/MockStorageInterface.java | 661 ----
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 215 --
.../fs/azure/NativeAzureFileSystemBaseTest.java | 1731 ---------
.../apache/hadoop/fs/azure/TestBlobMetadata.java | 269 --
.../fs/azure/TestBlobOperationDescriptor.java | 303 --
.../fs/azure/TestClientThrottlingAnalyzer.java | 167 -
.../fs/azure/TestKeyPageBlobDirectories.java | 169 -
.../TestNativeAzureFileSystemAuthorization.java | 2113 ----------
.../TestNativeAzureFileSystemBlockCompaction.java | 265 --
.../TestNativeAzureFileSystemConcurrency.java | 181 -
.../TestNativeAzureFileSystemContractMocked.java | 66 -
.../TestNativeAzureFileSystemFileNameCheck.java | 131 -
.../fs/azure/TestNativeAzureFileSystemMocked.java | 67 -
.../TestNativeAzureFileSystemOperationsMocked.java | 89 -
.../TestNativeAzureFileSystemUploadLogic.java | 181 -
.../fs/azure/TestOutOfBandAzureBlobOperations.java | 170 -
.../fs/azure/TestShellDecryptionKeyProvider.java | 81 -
.../fs/azure/TestSyncableDataOutputStream.java | 68 -
.../org/apache/hadoop/fs/azure/TestWasbFsck.java | 134 -
.../hadoop/fs/azure/TestWasbInitFailure.java | 71 +
.../contract/ITestAzureNativeContractAppend.java | 41 -
.../contract/ITestAzureNativeContractCreate.java | 34 -
.../contract/ITestAzureNativeContractDelete.java | 33 -
.../contract/ITestAzureNativeContractDistCp.java | 49 -
.../ITestAzureNativeContractGetFileStatus.java | 35 -
.../contract/ITestAzureNativeContractMkdir.java | 33 -
.../contract/ITestAzureNativeContractOpen.java | 34 -
.../contract/ITestAzureNativeContractRename.java | 34 -
.../contract/ITestAzureNativeContractSeek.java | 34 -
.../contract/NativeAzureFileSystemContract.java | 49 -
.../azure/integration/AbstractAzureScaleTest.java | 71 -
.../fs/azure/integration/AzureTestConstants.java | 191 -
.../fs/azure/integration/AzureTestUtils.java | 557 ---
.../azure/integration/CleanupTestContainers.java | 87 -
.../fs/azure/integration/ITestAzureHugeFiles.java | 459 ---
.../apache/hadoop/fs/azure/integration/Sizes.java | 43 -
.../fs/azure/metrics/AzureMetricsTestUtil.java | 83 -
.../ITestAzureFileSystemInstrumentation.java | 546 ---
.../azure/metrics/TestBandwidthGaugeUpdater.java | 78 -
.../TestNativeAzureFileSystemMetricsSystem.java | 100 -
.../fs/azure/metrics/TestRollingWindowAverage.java | 42 -
.../org/apache/hadoop/fs/azure/package-info.java} | 14 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 45 -
.../hadoop/fs/azurebfs/AbstractAbfsScaleTest.java | 12 +-
.../hadoop/fs/azurebfs/ITestAbfsHugeFiles.java | 10 +-
.../ITestAzureBlobFileSystemRandomRead.java | 103 -
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2044 ----------
.../azurebfs/commit/AbstractAbfsClusterITest.java | 6 +-
.../commit/ITestAbfsLoadManifestsStage.java | 5 +-
.../azurebfs/constants/TestConfigurationKeys.java | 3 +
.../ITestAbfsFileSystemContractDistCp.java | 6 +-
.../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 194 +-
.../hadoop-azure/src/test/resources/wasb.xml | 169 -
160 files changed, 402 insertions(+), 41137 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index b407a6f3adc..689cf1c8c1c 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -26,7 +26,7 @@
<name>Apache Hadoop Azure support</name>
<description>
This module contains code to support integration with Azure.
- Currently this consists of a filesystem client to read data from
+ Currently, this consists of Azure Blob File System client to read data from
and write data to Azure Storage.
</description>
<packaging>jar</packaging>
@@ -65,6 +65,7 @@
<effort>Max</effort>
</configuration>
</plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -89,7 +90,7 @@
</executions>
</plugin>
- <plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
@@ -109,7 +110,6 @@
</plugins>
-
<!--
The following is to suppress a m2e warning in eclipse
(m2e doesn't know how to handle maven-enforcer:enforce, so we have to tell m2e to ignore it)
@@ -153,7 +153,6 @@
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
-
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -244,8 +243,8 @@
except when using this committer, so it only tagged as
"provided".
It is not exported as a transitive dependency of the JAR.
- Applications which wish to to use the manifest committer
- will need to explicity add the mapreduce JAR to their classpath.
+ Applications which wish to use the manifest committer
+ will need to explicitly add the mapreduce JAR to their classpath.
This is already done by MapReduce and Spark.
-->
<dependency>
@@ -253,6 +252,7 @@
<artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>
+
<!--
These are only added to the classpath for tests,
and are not exported by the test JAR as transitive
@@ -264,41 +264,48 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
<scope>test</scope>
<type>jar</type>
</dependency>
+
<!-- artifacts needed to bring up a Mini MR Yarn cluster-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
@@ -329,37 +336,44 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<scope>test</scope>
</dependency>
+
<!-- Used to create SSL certs for a secure Keystore -->
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk18on</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
@@ -368,169 +382,6 @@
</dependencies>
<profiles>
- <profile>
- <id>parallel-tests-wasb</id>
- <activation>
- <property>
- <name>parallel-tests</name>
- <value>wasb</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>parallel-tests-createdir</id>
- <goals>
- <goal>parallel-tests-createdir</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <executions>
- <execution>
- <id>default-test</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <forkCount>${testsThreadCount}</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- <!-- http connection pool size -->
- <http.maxConnections>${http.maxConnections}</http.maxConnections> </systemPropertyVariables>
- <includes>
- <include>**/azure/Test*.java</include>
- <include>**/azure/**/Test*.java</include>
- </includes>
- <excludes>
- <exclude>**/azure/**/TestRollingWindowAverage*.java</exclude>
- </excludes>
- </configuration>
- </execution>
- <execution>
- <id>serialized-test-wasb</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <forkCount>1</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- <!-- http connection pool size -->
- <http.maxConnections>${http.maxConnections}</http.maxConnections>
- </systemPropertyVariables>
- <includes>
- <include>**/azure/**/TestRollingWindowAverage*.java</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <executions>
- <execution>
- <id>default-integration-test-wasb</id>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- <configuration>
- <forkCount>${testsThreadCount}</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <!-- Tell tests that they are being executed in parallel -->
- <test.parallel.execution>true</test.parallel.execution>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-
- <!-- Due to a Maven quirk, setting this to just -->
- <!-- surefire.forkNumber won't do the parameter -->
- <!-- substitution. Putting a prefix in front of it like -->
- <!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <!-- Propagate scale parameters -->
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- </systemPropertyVariables>
- <!-- Some tests cannot run in parallel-->
- <includes>
- <include>**/azure/ITest*.java</include>
- <include>**/azure/**/ITest*.java</include>
- </includes>
- <excludes>
- <exclude>**/azure/ITestNativeFileSystemStatistics.java</exclude>
- </excludes>
- </configuration>
- </execution>
- <!-- Do a sequential run for tests that cannot handle -->
- <!-- parallel execution. -->
- <execution>
- <id>sequential-integration-tests-wasb</id>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- <configuration>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.parallel.execution>false</test.parallel.execution>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- </systemPropertyVariables>
- <includes>
- <include>**/azure/ITestNativeFileSystemStatistics.java</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
<profile>
<id>parallel-tests-abfs</id>
@@ -764,36 +615,6 @@
<includes>
<include>**/Test*.java</include>
</includes>
- <excludes>
- <exclude>**/TestRollingWindowAverage*.java</exclude>
- </excludes>
- </configuration>
- </execution>
- <execution>
- <id>serialized-test</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <forkCount>1</forkCount>
- <reuseForks>false</reuseForks>
- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
- <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
- <systemPropertyVariables>
- <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
- <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
- <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
- <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
- <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
- <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
- <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
- <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
- <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
- </systemPropertyVariables>
- <includes>
- <include>**/TestRollingWindowAverage*.java</include>
- </includes>
</configuration>
</execution>
</executions>
@@ -850,18 +671,7 @@
<include>**/ITest*.java</include>
</includes>
<excludes>
- <exclude>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</exclude>
- <exclude>**/ITestFileSystemOperationsWithThreads.java</exclude>
- <exclude>**/ITestOutOfBandAzureBlobOperationsLive.java</exclude>
- <exclude>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</exclude>
- <exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude>
- <exclude>**/ITestNativeAzureFileSystemLive.java</exclude>
- <exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude>
<exclude>**/ITestAzureBlobFileSystemRandomRead.java</exclude>
- <exclude>**/ITestWasbRemoteCallHelper.java</exclude>
- <exclude>**/ITestBlockBlobInputStream.java</exclude>
- <exclude>**/ITestWasbAbfsCompatibility.java</exclude>
- <exclude>**/ITestNativeFileSystemStatistics.java</exclude>
</excludes>
</configuration>
</execution>
@@ -884,20 +694,10 @@
<fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
<fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
<!-- http connection pool size -->
- <http.maxConnections>${http.maxConnections}</http.maxConnections> </systemPropertyVariables>
+ <http.maxConnections>${http.maxConnections}</http.maxConnections>
+ </systemPropertyVariables>
<includes>
- <include>**/ITestWasbAbfsCompatibility.java</include>
- <include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
- <include>**/ITestFileSystemOperationsWithThreads.java</include>
- <include>**/ITestOutOfBandAzureBlobOperationsLive.java</include>
- <include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include>
- <include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include>
- <include>**/ITestNativeAzureFileSystemLive.java</include>
- <include>**/ITestNativeAzureFSPageBlobLive.java</include>
<include>**/ITestAzureBlobFileSystemRandomRead.java</include>
- <include>**/ITestWasbRemoteCallHelper.java</include>
- <include>**/ITestBlockBlobInputStream.java</include>
- <include>**/ITestNativeFileSystemStatistics.java</include>
</includes>
</configuration>
</execution>
@@ -957,5 +757,6 @@
<fs.azure.scale.test.enabled>true</fs.azure.scale.test.enabled>
</properties>
</profile>
+
</profiles>
</project>
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java
deleted file mode 100644
index 48ec064d5fa..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Thrown if there is a problem communicating with Azure Storage service.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class AzureException extends IOException {
- private static final long serialVersionUID = 1L;
-
- public AzureException(String message) {
- super(message);
- }
-
- public AzureException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public AzureException(Throwable t) {
- super(t);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java
deleted file mode 100644
index 98d28d1ea42..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureFileSystemThreadPoolExecutor.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-
-class AzureFileSystemThreadPoolExecutor {
-
- public static final Logger LOG = LoggerFactory.getLogger(AzureFileSystemThreadPoolExecutor.class);
-
- /*
- * Number of threads to keep in the pool.
- */
- private int threadCount;
-
- /*
- * Prefix to be used for naming threads.
- */
- private String threadNamePrefix;
-
- /*
- * File system operation like delete/rename. Used for logging purpose.
- */
- private String operation;
-
- /*
- * Source blob key used for file operation. Used for logging purpose.
- */
- private String key;
-
- /*
- * Configuration name for recommendations. Used for logging purpose.
- */
- private String config;
-
- /**
- * Creates a new AzureFileSystemThreadPoolExecutor object.
- *
- * @param threadCount
- * Number of threads to be used after reading user configuration.
- * @param threadNamePrefix
- * Prefix to be used to name threads for the file operation.
- * @param operation
- * File system operation like delete/rename. Used for logging purpose.
- * @param key
- * Source blob key used for file operation. Used for logging purpose.
- * @param config
- * Configuration name for recommendations. Used for logging purpose.
- */
- public AzureFileSystemThreadPoolExecutor(int threadCount, String threadNamePrefix,
- String operation, String key, String config) {
- this.threadCount = threadCount;
- this.threadNamePrefix = threadNamePrefix;
- this.operation = operation;
- this.key = key;
- this.config = config;
- }
-
- /**
- * Gets a new thread pool
- * @param threadCount
- * Number of threads to keep in the pool.
- * @param threadNamePrefix
- * Prefix to be used for naming threads.
- *
- * @return
- * Returns a new thread pool.
- */
- @VisibleForTesting
- ThreadPoolExecutor getThreadPool(int threadCount) throws Exception {
- return new ThreadPoolExecutor(threadCount, threadCount, 2, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new AzureFileSystemThreadFactory(this.threadNamePrefix));
- }
-
- /**
- * Execute the file operation parallel using threads. All threads works on a
- * single working set of files stored in input 'contents'. The synchronization
- * between multiple threads is achieved through retrieving atomic index value
- * from the array. Once thread gets the index, it retrieves the file and initiates
- * the file operation. The advantage with this method is that file operations
- * doesn't get serialized due to any thread. Also, the input copy is not changed
- * such that caller can reuse the list for other purposes.
- *
- * This implementation also considers that failure of operation on single file
- * is considered as overall operation failure. All threads bail out their execution
- * as soon as they detect any single thread either got exception or operation is failed.
- *
- * @param contents
- * List of blobs on which operation to be done.
- * @param threadOperation
- * The actual operation to be executed by each thread on a file.
- *
- * @param operationStatus
- * Returns true if the operation is success, false if operation is failed.
- * @throws IOException
- *
- */
- boolean executeParallel(FileMetadata[] contents, AzureFileSystemThreadTask threadOperation) throws IOException {
-
- boolean operationStatus = false;
- boolean threadsEnabled = false;
- int threadCount = this.threadCount;
- ThreadPoolExecutor ioThreadPool = null;
-
- // Start time for file operation
- long start = Time.monotonicNow();
-
- // If number of files are less then reduce threads to file count.
- threadCount = Math.min(contents.length, threadCount);
-
- if (threadCount > 1) {
- try {
- ioThreadPool = getThreadPool(threadCount);
- threadsEnabled = true;
- } catch(Exception e) {
- // The possibility of this scenario is very remote. Added this code as safety net.
- LOG.warn("Failed to create thread pool with threads {} for operation {} on blob {}."
- + " Use config {} to set less number of threads. Setting config value to <= 1 will disable threads.",
- threadCount, operation, key, config);
- }
- } else {
- LOG.warn("Disabling threads for {} operation as thread count {} is <= 1", operation, threadCount);
- }
-
- if (threadsEnabled) {
- LOG.debug("Using thread pool for {} operation with threads {}", operation, threadCount);
- boolean started = false;
- AzureFileSystemThreadRunnable runnable = new AzureFileSystemThreadRunnable(contents, threadOperation, operation);
-
- // Don't start any new requests if there is an exception from any one thread.
- for (int i = 0; i < threadCount && runnable.lastException == null && runnable.operationStatus; i++)
- {
- try {
- ioThreadPool.execute(runnable);
- started = true;
- } catch (RejectedExecutionException ex) {
- // If threads can't be scheduled then report error and move ahead with next thread.
- // Don't fail operation due to this issue.
- LOG.error("Rejected execution of thread for {} operation on blob {}."
- + " Continuing with existing threads. Use config {} to set less number of threads"
- + " to avoid this error", operation, key, config);
- }
- }
-
- // Stop accepting any new execute requests.
- ioThreadPool.shutdown();
-
- try {
- // Wait for threads to terminate. Keep time out as large value
- ioThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
- } catch(InterruptedException intrEx) {
- // If current thread got interrupted then shutdown all threads now.
- ioThreadPool.shutdownNow();
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- LOG.error("Threads got interrupted {} blob operation for {} "
- , operation, key);
- }
-
- int threadsNotUsed = threadCount - runnable.threadsUsed.get();
- if (threadsNotUsed > 0) {
- LOG.warn("{} threads not used for {} operation on blob {}", threadsNotUsed, operation, key);
- }
-
- if (!started) {
- // No threads started. Fall back to serial mode.
- threadsEnabled = false;
- LOG.info("Not able to schedule threads to {} blob {}. Fall back to {} blob serially."
- , operation, key, operation);
- } else {
- IOException lastException = runnable.lastException;
-
- // There are no exceptions from threads and no operation failures. Consider this scenario
- // as failure only if file operations are not done on all files.
- if (lastException == null && runnable.operationStatus && runnable.filesProcessed.get() < contents.length) {
- LOG.error("{} failed as operation on subfolders and files failed.", operation);
- lastException = new IOException(operation + " failed as operation on subfolders and files failed.");
- }
-
- if (lastException != null) {
- // Threads started and executed. One or more threads seems to have hit exception.
- // Raise the same exception.
- throw lastException;
- }
-
- operationStatus = runnable.operationStatus;
- }
- }
-
- if (!threadsEnabled) {
- // No threads. Serialize the operation. Clear any last exceptions.
- LOG.debug("Serializing the {} operation", operation);
- for (int i = 0; i < contents.length; i++) {
- if (!threadOperation.execute(contents[i])) {
- LOG.warn("Failed to {} file {}", operation, contents[i]);
- return false;
- }
- }
-
- // Operation is success
- operationStatus = true;
- }
-
- // Find the duration of time taken for file operation
- long end = Time.monotonicNow();
- LOG.info("Time taken for {} operation is: {} ms with threads: {}", operation, (end - start), threadCount);
-
- return operationStatus;
- }
-
- /**
- * A ThreadFactory for Azure File operation threads with meaningful names helpful
- * for debugging purposes.
- */
- static class AzureFileSystemThreadFactory implements ThreadFactory {
-
- private String threadIdPrefix = "AzureFileSystemThread";
-
- /**
- * Atomic integer to provide thread id for thread names.
- */
- private AtomicInteger threadSequenceNumber = new AtomicInteger(0);
-
- public AzureFileSystemThreadFactory(String prefix) {
- threadIdPrefix = prefix;
- }
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new SubjectInheritingThread(r);
-
- // Use current thread name as part in naming thread such that use of
- // same file system object will have unique names.
- t.setName(String.format("%s-%s-%d", threadIdPrefix, Thread.currentThread().getName(),
- threadSequenceNumber.getAndIncrement()));
- return t;
- }
-
- }
-
- static class AzureFileSystemThreadRunnable implements Runnable {
-
- // Tracks if any thread has raised exception.
- private volatile IOException lastException = null;
-
- // Tracks if any thread has failed execution.
- private volatile boolean operationStatus = true;
-
- // Atomic tracker to retrieve index of next file to be processed
- private AtomicInteger fileIndex = new AtomicInteger(0);
-
- // Atomic tracker to count number of files successfully processed
- private AtomicInteger filesProcessed = new AtomicInteger(0);
-
- // Atomic tracker to retrieve number of threads used to do at least one file operation.
- private AtomicInteger threadsUsed = new AtomicInteger(0);
-
- // Type of file system operation
- private String operation = "Unknown";
-
- // List of files to be processed.
- private final FileMetadata[] files;
-
- // Thread task which encapsulates the file system operation work on a file.
- private AzureFileSystemThreadTask task;
-
- public AzureFileSystemThreadRunnable(final FileMetadata[] files,
- AzureFileSystemThreadTask task, String operation) {
- this.operation = operation;
- this.files = files;
- this.task = task;
- }
-
- @Override
- public void run() {
- long start = Time.monotonicNow();
- int currentIndex;
- int processedFilesCount = 0;
-
- while ((currentIndex = fileIndex.getAndIncrement()) < files.length) {
- processedFilesCount++;
- FileMetadata file = files[currentIndex];
-
- try {
- // Execute the file operation.
- if (!task.execute(file)) {
- LOG.error("{} operation failed for file {}",
- this.operation, file.getKey());
- operationStatus = false;
- } else {
- filesProcessed.getAndIncrement();
- }
- } catch (Exception e) {
- LOG.error("Encountered Exception for {} operation for file {}",
- this.operation, file.getKey());
- lastException = new IOException("Encountered Exception for "
- + this.operation + " operation for file " + file.getKey(), e);
- }
-
- // If any thread has seen exception or operation failed then we
- // don't have to process further.
- if (lastException != null || !operationStatus) {
- LOG.warn("Terminating execution of {} operation now as some other thread"
- + " already got exception or operation failed", this.operation, file.getKey());
- break;
- }
- }
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to process {} files count for {} operation: {} ms",
- processedFilesCount, this.operation, (end - start));
- if (processedFilesCount > 0) {
- threadsUsed.getAndIncrement();
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java
deleted file mode 100644
index 4c52ef0e931..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureLinkedStack.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * A simple generic stack implementation using linked lists. The stack
- * implementation has five main operations:
- * <ul>
- * <li>push -- adds an element to the top of the stack</li>
- * <li>pop -- removes an element from the top of the stack and returns a
- * reference to it</li>
- * <li>peek -- peek returns an element from the top of the stack without
- * removing it</li>
- * <li>isEmpty -- tests whether the stack is empty</li>
- * <li>size -- returns the size of the stack</li>
- * <li>toString -- returns a string representation of the stack.</li>
- * </ul>
- */
-
-public class AzureLinkedStack<E> {
- /*
- * Linked node for Azure stack collection.
- */
- private static class AzureLinkedNode<E> {
- private E element; // Linked element on the list.
- private AzureLinkedNode<E> next;// Reference to the next linked element on
- // list.
-
- /*
- * The constructor builds the linked node with no successor
- *
- * @param element : The value of the element to be stored with this node.
- */
- private AzureLinkedNode(E anElement) {
- element = anElement;
- next = null;
- }
-
- /*
- * Constructor builds a linked node with a specified successor. The
- * successor may be null.
- *
- * @param anElement : new element to be created.
- *
- * @param nextElement: successor to the new element.
- */
- private AzureLinkedNode(E anElement, AzureLinkedNode<E> nextElement) {
- element = anElement;
- next = nextElement;
- }
-
- /*
- * Get the element stored in the linked node.
- *
- * @return E : element stored in linked node.
- */
- private E getElement() {
- return element;
- }
-
- /*
- * Get the successor node to the element.
- *
- * @return E : reference to the succeeding node on the list.
- */
- private AzureLinkedNode<E> getNext() {
- return next;
- }
- }
-
- private int count; // The number of elements stored on the stack.
- private AzureLinkedNode<E> top; // Top of the stack.
-
- /*
- * Constructor creating an empty stack.
- */
- public AzureLinkedStack() {
- // Simply initialize the member variables.
- //
- count = 0;
- top = null;
- }
-
- /*
- * Adds an element to the top of the stack.
- *
- * @param element : element pushed to the top of the stack.
- */
- public void push(E element) {
- // Create a new node containing a reference to be placed on the stack.
- // Set the next reference to the new node to point to the current top
- // of the stack. Set the top reference to point to the new node. Finally
- // increment the count of nodes on the stack.
- //
- AzureLinkedNode<E> newNode = new AzureLinkedNode<E>(element, top);
- top = newNode;
- count++;
- }
-
- /*
- * Removes the element at the top of the stack and returns a reference to it.
- *
- * @return E : element popped from the top of the stack.
- *
- * @throws Exception on pop from an empty stack.
- */
- public E pop() throws Exception {
- // Make sure the stack is not empty. If it is empty, throw a StackEmpty
- // exception.
- //
- if (isEmpty()) {
- throw new Exception("AzureStackEmpty");
- }
-
- // Set a temporary reference equal to the element at the top of the stack,
- // decrement the count of elements and return reference to the temporary.
- //
- E element = top.getElement();
- top = top.getNext();
- count--;
-
- // Return the reference to the element that was at the top of the stack.
- //
- return element;
- }
-
- /*
- * Return the top element of the stack without removing it.
- *
- * @return E
- *
- * @throws Exception on peek into an empty stack.
- */
- public E peek() throws Exception {
- // Make sure the stack is not empty. If it is empty, throw a StackEmpty
- // exception.
- //
- if (isEmpty()) {
- throw new Exception("AzureStackEmpty");
- }
-
- // Set a temporary reference equal to the element at the top of the stack
- // and return the temporary.
- //
- E element = top.getElement();
- return element;
- }
-
- /*
- * Determines whether the stack is empty
- *
- * @return boolean true if the stack is empty and false otherwise.
- */
- public boolean isEmpty() {
- if (0 == size()) {
- // Zero-sized stack so the stack is empty.
- //
- return true;
- }
-
- // The stack is not empty.
- //
- return false;
- }
-
- /*
- * Determines the size of the stack
- *
- * @return int: Count of the number of elements in the stack.
- */
- public int size() {
- return count;
- }
-
- /*
- * Returns a string representation of the stack.
- *
- * @return String String representation of all elements in the stack.
- */
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
-
- AzureLinkedNode<E> current = top;
- for (int i = 0; i < size(); i++) {
- E element = current.getElement();
- sb.append(element.toString());
- current = current.getNext();
-
- // Insert commas between strings except after the last string.
- //
- if (size() - 1 > i) {
- sb.append(", ");
- }
- }
-
- // Return the string.
- //
- return sb.toString();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
deleted file mode 100644
index 6e50cfc627e..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ /dev/null
@@ -1,3105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.PATH_DELIMITER;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobContainerWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-import org.apache.hadoop.fs.azure.metrics.BandwidthGaugeUpdater;
-import org.apache.hadoop.fs.azure.metrics.ErrorMetricUpdater;
-import org.apache.hadoop.fs.azure.metrics.ResponseReceivedMetricUpdater;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.VersionInfo;
-import org.eclipse.jetty.util.ajax.JSON;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryExponentialRetry;
-import com.microsoft.azure.storage.RetryNoRetry;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
-import com.microsoft.azure.storage.StorageErrorCodeStrings;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.Constants;
-import com.microsoft.azure.storage.StorageEvent;
-import com.microsoft.azure.storage.core.BaseRequest;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.BlobProperties;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlobType;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CopyStatus;
-import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.core.Utility;
-
-/**
- * Core implementation of Windows Azure Filesystem for Hadoop.
- * Provides the bridging logic between Hadoop's abstract filesystem and Azure Storage
- *
- */
-@InterfaceAudience.Private
-@VisibleForTesting
-public class AzureNativeFileSystemStore implements NativeFileSystemStore {
-
- /**
- * Configuration knob on whether we do block-level MD5 validation on
- * upload/download.
- */
- static final String KEY_CHECK_BLOCK_MD5 = "fs.azure.check.block.md5";
- /**
- * Configuration knob on whether we store blob-level MD5 on upload.
- */
- static final String KEY_STORE_BLOB_MD5 = "fs.azure.store.blob.md5";
- static final String DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME = "storageemulator";
- static final String STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.storage.emulator.account.name";
-
- /**
- * Configuration for User-Agent field.
- */
- static final String USER_AGENT_ID_KEY = "fs.azure.user.agent.prefix";
- static final String USER_AGENT_ID_DEFAULT = "unknown";
-
- public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class);
-
- private StorageInterface storageInteractionLayer;
- private CloudBlobDirectoryWrapper rootDirectory;
- private CloudBlobContainerWrapper container;
-
- // Constants local to this class.
- //
- private static final String KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider.";
- private static final String KEY_ACCOUNT_SAS_PREFIX = "fs.azure.sas.";
-
- // note: this value is not present in core-default.xml as our real default is
- // computed as min(2*cpu,8)
- private static final String KEY_CONCURRENT_CONNECTION_VALUE_OUT = "fs.azure.concurrentRequestCount.out";
-
- private static final String HADOOP_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size";
- private static final String KEY_STREAM_MIN_READ_SIZE = "fs.azure.read.request.size";
- private static final String KEY_STORAGE_CONNECTION_TIMEOUT = "fs.azure.storage.timeout";
- private static final String KEY_WRITE_BLOCK_SIZE = "fs.azure.write.request.size";
- @VisibleForTesting
- static final String KEY_INPUT_STREAM_VERSION = "fs.azure.input.stream.version.for.internal.use.only";
-
- // Property controlling whether to allow reads on blob which are concurrently
- // appended out-of-band.
- private static final String KEY_READ_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append";
-
- // Configurable throttling parameter properties. These properties are located
- // in the core-site.xml configuration file.
- private static final String KEY_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval";
- private static final String KEY_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval";
- private static final String KEY_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval";
- private static final String KEY_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries";
-
- private static final String KEY_COPYBLOB_MIN_BACKOFF_INTERVAL =
- "fs.azure.io.copyblob.retry.min.backoff.interval";
- private static final String KEY_COPYBLOB_MAX_BACKOFF_INTERVAL =
- "fs.azure.io.copyblob.retry.max.backoff.interval";
- private static final String KEY_COPYBLOB_BACKOFF_INTERVAL =
- "fs.azure.io.copyblob.retry.backoff.interval";
- private static final String KEY_COPYBLOB_MAX_IO_RETRIES =
- "fs.azure.io.copyblob.retry.max.retries";
-
- private static final String KEY_SELF_THROTTLE_ENABLE = "fs.azure.selfthrottling.enable";
- private static final String KEY_SELF_THROTTLE_READ_FACTOR = "fs.azure.selfthrottling.read.factor";
- private static final String KEY_SELF_THROTTLE_WRITE_FACTOR = "fs.azure.selfthrottling.write.factor";
-
- private static final String KEY_AUTO_THROTTLE_ENABLE = "fs.azure.autothrottling.enable";
-
- private static final String KEY_ENABLE_STORAGE_CLIENT_LOGGING = "fs.azure.storage.client.logging";
-
- /**
- * Configuration keys to identify if WASB needs to run in Secure mode. In Secure mode
- * all interactions with Azure storage is performed using SAS uris. There are two sub modes
- * within the Secure mode , one is remote SAS key mode where the SAS keys are generated
- * from a remote process and local mode where SAS keys are generated within WASB.
- */
- @VisibleForTesting
- public static final String KEY_USE_SECURE_MODE = "fs.azure.secure.mode";
-
- /**
- * By default the SAS Key mode is expected to run in Romote key mode. This flags sets it
- * to run on the local mode.
- */
- public static final String KEY_USE_LOCAL_SAS_KEY_MODE = "fs.azure.local.sas.key.mode";
-
- /**
- * Config to control case sensitive metadata key checks/retrieval. If this
- * is false, blob metadata keys will be treated case insensitive.
- */
- private static final String KEY_BLOB_METADATA_KEY_CASE_SENSITIVE = "fs.azure.blob.metadata.key.case.sensitive";
- private static final String PERMISSION_METADATA_KEY = "hdi_permission";
- private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission";
- private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder";
- private static final String OLD_IS_FOLDER_METADATA_KEY = "asv_isfolder";
- static final String VERSION_METADATA_KEY = "hdi_version";
- static final String OLD_VERSION_METADATA_KEY = "asv_version";
- static final String FIRST_WASB_VERSION = "2013-01-01";
- static final String CURRENT_WASB_VERSION = "2013-09-01";
- static final String LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY = "hdi_tmpupload";
- static final String OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY = "asv_tmpupload";
-
- /**
- * Configuration key to indicate the set of directories in WASB where we
- * should store files as page blobs instead of block blobs.
- *
- * Entries should be plain directory names (i.e. not URIs) with no leading or
- * trailing slashes. Delimit the entries with commas.
- */
- public static final String KEY_PAGE_BLOB_DIRECTORIES =
- "fs.azure.page.blob.dir";
- /**
- * The set of directories where we should store files as page blobs.
- */
- private Set<String> pageBlobDirs;
-
- /**
- * Configuration key to indicate the set of directories in WASB where we
- * should store files as block blobs with block compaction enabled.
- *
- * Entries can be directory paths relative to the container (e.g. "/path") or
- * fully qualified wasb:// URIs (e.g.
- * wasb://container@example.blob.core.windows.net/path)
- */
- public static final String KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES =
- "fs.azure.block.blob.with.compaction.dir";
-
- /**
- * The set of directories where we should store files as block blobs with
- * block compaction enabled.
- */
- private Set<String> blockBlobWithCompationDirs;
-
- /**
- * Configuration key to indicate the set of directories in WASB where
- * we should do atomic folder rename synchronized with createNonRecursive.
- */
- public static final String KEY_ATOMIC_RENAME_DIRECTORIES =
- "fs.azure.atomic.rename.dir";
-
- /**
- * Configuration key to enable flat listing of blobs. This config is useful
- * only if listing depth is AZURE_UNBOUNDED_DEPTH.
- */
- public static final String KEY_ENABLE_FLAT_LISTING = "fs.azure.flatlist.enable";
-
- /**
- * Optional config to enable a lock free pread which will bypass buffer in
- * BlockBlobInputStream.
- * This is not a config which can be set at cluster level. It can be used as
- * an option on FutureDataInputStreamBuilder.
- * @see FileSystem#openFile(org.apache.hadoop.fs.Path)
- */
- public static final String FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE =
- "fs.azure.block.blob.buffered.pread.disable";
-
- /**
- * The set of directories where we should apply atomic folder rename
- * synchronized with createNonRecursive.
- */
- private Set<String> atomicRenameDirs;
-
- private static final String HTTP_SCHEME = "http";
- private static final String HTTPS_SCHEME = "https";
- private static final String WASB_AUTHORITY_DELIMITER = "@";
- private static final char ASTERISK_SYMBOL = '*';
- private static final String AZURE_ROOT_CONTAINER = "$root";
-
- private static final int DEFAULT_CONCURRENT_WRITES = 8;
-
- private static final Charset METADATA_ENCODING = StandardCharsets.UTF_8;
-
- // Concurrent reads reads of data written out of band are disable by default.
- //
- private static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false;
-
- // Default block sizes
- public static final int DEFAULT_DOWNLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
- public static final int DEFAULT_UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
- public static final long DEFAULT_HADOOP_BLOCK_SIZE = 512 * 1024 * 1024L;
-
- private static final int DEFAULT_INPUT_STREAM_VERSION = 2;
-
- // Retry parameter defaults.
- //
-
- private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s
- private static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s
- private static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s
- private static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
-
- private static final int DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL = 3 * 1000;
- private static final int DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL = 90 * 1000;
- private static final int DEFAULT_COPYBLOB_BACKOFF_INTERVAL = 30 * 1000;
- private static final int DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS = 15;
-
- // Self-throttling defaults. Allowed range = (0,1.0]
- // Value of 1.0 means no self-throttling.
- // Value of x means process data at factor x of unrestricted rate
- private static final boolean DEFAULT_SELF_THROTTLE_ENABLE = true;
- private static final float DEFAULT_SELF_THROTTLE_READ_FACTOR = 1.0f;
- private static final float DEFAULT_SELF_THROTTLE_WRITE_FACTOR = 1.0f;
-
- private static final boolean DEFAULT_AUTO_THROTTLE_ENABLE = false;
-
- private static final int STORAGE_CONNECTION_TIMEOUT_DEFAULT = 90;
-
- /**
- * Default values to control SAS Key mode.
- * By default we set the values to false.
- */
- public static final boolean DEFAULT_USE_SECURE_MODE = false;
- private static final boolean DEFAULT_USE_LOCAL_SAS_KEY_MODE = false;
-
- /**
- * Enable flat listing of blobs as default option. This is useful only if
- * listing depth is AZURE_UNBOUNDED_DEPTH.
- */
- public static final boolean DEFAULT_ENABLE_FLAT_LISTING = false;
-
- /**
- * MEMBER VARIABLES
- */
-
- private URI sessionUri;
- private Configuration sessionConfiguration;
- private int concurrentWrites = DEFAULT_CONCURRENT_WRITES;
- private boolean isAnonymousCredentials = false;
- // Set to true if we are connecting using shared access signatures.
- private boolean connectingUsingSAS = false;
- private AzureFileSystemInstrumentation instrumentation;
- private BandwidthGaugeUpdater bandwidthGaugeUpdater;
- private static final JSON PERMISSION_JSON_SERIALIZER = createPermissionJsonSerializer();
-
- private boolean suppressRetryPolicy = false;
- private boolean canCreateOrModifyContainer = false;
- private ContainerState currentKnownContainerState = ContainerState.Unknown;
- private final Object containerStateLock = new Object();
-
- private boolean tolerateOobAppends = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND;
-
- private long hadoopBlockSize = DEFAULT_HADOOP_BLOCK_SIZE;
- private int downloadBlockSizeBytes = DEFAULT_DOWNLOAD_BLOCK_SIZE;
- private int uploadBlockSizeBytes = DEFAULT_UPLOAD_BLOCK_SIZE;
- private int inputStreamVersion = DEFAULT_INPUT_STREAM_VERSION;
-
- // Bandwidth throttling exponential back-off parameters
- //
- private int minBackoff; // the minimum back-off interval (ms) between retries.
- private int maxBackoff; // the maximum back-off interval (ms) between retries.
- private int deltaBackoff; // the back-off interval (ms) between retries.
- private int maxRetries; // the maximum number of retry attempts.
-
- // Self-throttling parameters
- private boolean selfThrottlingEnabled;
- private float selfThrottlingReadFactor;
- private float selfThrottlingWriteFactor;
-
- private boolean autoThrottlingEnabled;
-
- private TestHookOperationContext testHookOperationContext = null;
-
- // Set if we're running against a storage emulator..
- private boolean isStorageEmulator = false;
-
- // Configs controlling WASB SAS key mode.
- private boolean useSecureMode = false;
- private boolean useLocalSasKeyMode = false;
-
- // User-Agent
- private String userAgentId;
-
- private String delegationToken;
-
- private boolean metadataKeyCaseSensitive;
-
- /** The error message template when container is not accessible. */
- public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
- + "account %s in the configuration, and its container %s is not "
- + "accessible using anonymous credentials. Please check if the container "
- + "exists first. If it is not publicly available, you have to provide "
- + "account credentials.";
-
- /**
- * A test hook interface that can modify the operation context we use for
- * Azure Storage operations, e.g. to inject errors.
- */
- @VisibleForTesting
- interface TestHookOperationContext {
- OperationContext modifyOperationContext(OperationContext original);
- }
-
- /**
- * Suppress the default retry policy for the Storage, useful in unit tests to
- * test negative cases without waiting forever.
- */
- @VisibleForTesting
- void suppressRetryPolicy() {
- suppressRetryPolicy = true;
- }
-
- /**
- * Add a test hook to modify the operation context we use for Azure Storage
- * operations.
- *
- * @param testHook
- * The test hook, or null to unset previous hooks.
- */
- @VisibleForTesting
- void addTestHookToOperationContext(TestHookOperationContext testHook) {
- this.testHookOperationContext = testHook;
- }
-
- /**
- * If we're asked by unit tests to not retry, set the retry policy factory in
- * the client accordingly.
- */
- private void suppressRetryPolicyInClientIfNeeded() {
- if (suppressRetryPolicy) {
- storageInteractionLayer.setRetryPolicyFactory(new RetryNoRetry());
- }
- }
-
- /**
- * Creates a JSON serializer that can serialize a PermissionStatus object into
- * the JSON string we want in the blob metadata.
- *
- * @return The JSON serializer.
- */
- private static JSON createPermissionJsonSerializer() {
- org.eclipse.jetty.util.log.Log.getProperties().setProperty("org.eclipse.jetty.util.log.announce", "false");
- JSON serializer = new JSON();
- serializer.addConvertor(PermissionStatus.class,
- new PermissionStatusJsonSerializer());
- return serializer;
- }
-
- /**
- * A converter for PermissionStatus to/from JSON as we want it in the blob
- * metadata.
- */
- private static class PermissionStatusJsonSerializer implements JSON.Convertor {
- private static final String OWNER_TAG = "owner";
- private static final String GROUP_TAG = "group";
- private static final String PERMISSIONS_TAG = "permissions";
-
- @Override
- public void toJSON(Object obj, JSON.Output out) {
- PermissionStatus permissionStatus = (PermissionStatus) obj;
- // Don't store group as null, just store it as empty string
- // (which is FileStatus behavior).
- String group = permissionStatus.getGroupName() == null ? ""
- : permissionStatus.getGroupName();
- out.add(OWNER_TAG, permissionStatus.getUserName());
- out.add(GROUP_TAG, group);
- out.add(PERMISSIONS_TAG, permissionStatus.getPermission().toString());
- }
-
- @Override
- public Object fromJSON(@SuppressWarnings("rawtypes") Map object) {
- return PermissionStatusJsonSerializer.fromJSONMap(object);
- }
-
- @SuppressWarnings("rawtypes")
- public static PermissionStatus fromJSONString(String jsonString) {
- // The JSON class can only find out about an object's class (and call me)
- // if we store the class name in the JSON string. Since I don't want to
- // do that (it's an implementation detail), I just deserialize as a
- // the default Map (JSON's default behavior) and parse that.
- return fromJSONMap((Map) PERMISSION_JSON_SERIALIZER.fromJSON(jsonString));
- }
-
- private static PermissionStatus fromJSONMap(
- @SuppressWarnings("rawtypes") Map object) {
- return new PermissionStatus((String) object.get(OWNER_TAG),
- (String) object.get(GROUP_TAG),
- // The initial - below is the Unix file type,
- // which FsPermission needs there but ignores.
- FsPermission.valueOf("-" + (String) object.get(PERMISSIONS_TAG)));
- }
- }
-
- @VisibleForTesting
- void setAzureStorageInteractionLayer(StorageInterface storageInteractionLayer) {
- this.storageInteractionLayer = storageInteractionLayer;
- }
-
- @VisibleForTesting
- public BandwidthGaugeUpdater getBandwidthGaugeUpdater() {
- return bandwidthGaugeUpdater;
- }
-
- /**
- * Check if concurrent reads and writes on the same blob are allowed.
- *
- * @return true if concurrent reads and OOB writes has been configured, false
- * otherwise.
- */
- private boolean isConcurrentOOBAppendAllowed() {
- return tolerateOobAppends;
- }
-
- /**
- * Method for the URI and configuration object necessary to create a storage
- * session with an Azure session. It parses the scheme to ensure it matches
- * the storage protocol supported by this file system.
- *
- * @param uri - URI for target storage blob.
- * @param conf - reference to configuration object.
- * @param instrumentation - the metrics source that will keep track of operations here.
- *
- * @throws IllegalArgumentException if URI or job object is null, or invalid scheme.
- */
- @Override
- public void initialize(URI uri, Configuration conf, AzureFileSystemInstrumentation instrumentation)
- throws IllegalArgumentException, AzureException, IOException {
-
- if (null == instrumentation) {
- throw new IllegalArgumentException("Null instrumentation");
- }
- this.instrumentation = instrumentation;
-
- // Check that URI exists.
- //
- if (null == uri) {
- throw new IllegalArgumentException(
- "Cannot initialize WASB file system, URI is null");
- }
-
- // Check that configuration object is non-null.
- //
- if (null == conf) {
- throw new IllegalArgumentException(
- "Cannot initialize WASB file system, conf is null");
- }
-
- if (!conf.getBoolean(
- NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
- //If not skip azure metrics, create bandwidthGaugeUpdater
- this.bandwidthGaugeUpdater = new BandwidthGaugeUpdater(instrumentation);
- }
-
- // Incoming parameters validated. Capture the URI and the job configuration
- // object.
- //
- sessionUri = uri;
- sessionConfiguration = conf;
-
- useSecureMode = conf.getBoolean(KEY_USE_SECURE_MODE,
- DEFAULT_USE_SECURE_MODE);
- useLocalSasKeyMode = conf.getBoolean(KEY_USE_LOCAL_SAS_KEY_MODE,
- DEFAULT_USE_LOCAL_SAS_KEY_MODE);
-
- if (null == this.storageInteractionLayer) {
- if (!useSecureMode) {
- this.storageInteractionLayer = new StorageInterfaceImpl();
- } else {
- this.storageInteractionLayer = new SecureStorageInterfaceImpl(
- useLocalSasKeyMode, conf);
- }
- }
-
- // Configure Azure storage session.
- configureAzureStorageSession();
-
- // Start an Azure storage session.
- //
- createAzureStorageSession();
-
- // Extract the directories that should contain page blobs
- pageBlobDirs = getDirectorySet(KEY_PAGE_BLOB_DIRECTORIES);
- LOG.debug("Page blob directories: {}", setToString(pageBlobDirs));
-
- // User-agent
- userAgentId = conf.get(USER_AGENT_ID_KEY, USER_AGENT_ID_DEFAULT);
-
- // Extract the directories that should contain block blobs with compaction
- blockBlobWithCompationDirs = getDirectorySet(
- KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES);
- LOG.debug("Block blobs with compaction directories: {}",
- setToString(blockBlobWithCompationDirs));
-
- // Extract directories that should have atomic rename applied.
- atomicRenameDirs = getDirectorySet(KEY_ATOMIC_RENAME_DIRECTORIES);
- String hbaseRoot;
- try {
-
- // Add to this the hbase root directory, or /hbase is that is not set.
- hbaseRoot = verifyAndConvertToStandardFormat(
- sessionConfiguration.get("hbase.rootdir", "hbase"));
- if (hbaseRoot != null) {
- atomicRenameDirs.add(hbaseRoot);
- }
- } catch (URISyntaxException e) {
- LOG.warn("Unable to initialize HBase root as an atomic rename directory.");
- }
- LOG.debug("Atomic rename directories: {} ", setToString(atomicRenameDirs));
- metadataKeyCaseSensitive = conf
- .getBoolean(KEY_BLOB_METADATA_KEY_CASE_SENSITIVE, true);
- if (!metadataKeyCaseSensitive) {
- LOG.info("{} configured as false. Blob metadata will be treated case insensitive.",
- KEY_BLOB_METADATA_KEY_CASE_SENSITIVE);
- }
- }
-
- /**
- * Helper to format a string for log output from Set<String>
- */
- private String setToString(Set<String> set) {
- StringBuilder sb = new StringBuilder();
- int i = 1;
- for (String s : set) {
- sb.append("/" + s);
- if (i != set.size()) {
- sb.append(", ");
- }
- i++;
- }
- return sb.toString();
- }
-
- /**
- * Method to extract the account name from an Azure URI.
- *
- * @param uri
- * -- WASB blob URI
- * @returns accountName -- the account name for the URI.
- * @throws URISyntaxException
- * if the URI does not have an authority it is badly formed.
- */
- private String getAccountFromAuthority(URI uri) throws URISyntaxException {
-
- // Check to make sure that the authority is valid for the URI.
- //
- String authority = uri.getRawAuthority();
- if (null == authority) {
- // Badly formed or illegal URI.
- //
- throw new URISyntaxException(uri.toString(),
- "Expected URI with a valid authority");
- }
-
- // Check if authority container the delimiter separating the account name from the
- // the container.
- //
- if (!authority.contains(WASB_AUTHORITY_DELIMITER)) {
- return authority;
- }
-
- // Split off the container name and the authority.
- //
- String[] authorityParts = authority.split(WASB_AUTHORITY_DELIMITER, 2);
-
- // Because the string contains an '@' delimiter, a container must be
- // specified.
- //
- if (authorityParts.length < 2 || "".equals(authorityParts[0])) {
- // Badly formed WASB authority since there is no container.
- //
- final String errMsg = String
- .format(
- "URI '%s' has a malformed WASB authority, expected container name. "
- + "Authority takes the form wasb://[<container name>@]<account name>",
- uri.toString());
- throw new IllegalArgumentException(errMsg);
- }
-
- // Return with the account name. It is possible that this name is NULL.
- //
- return authorityParts[1];
- }
-
- /**
- * Method to extract the container name from an Azure URI.
- *
- * @param uri
- * -- WASB blob URI
- * @returns containerName -- the container name for the URI. May be null.
- * @throws URISyntaxException
- * if the uri does not have an authority it is badly formed.
- */
- private String getContainerFromAuthority(URI uri) throws URISyntaxException {
-
- // Check to make sure that the authority is valid for the URI.
- //
- String authority = uri.getRawAuthority();
- if (null == authority) {
- // Badly formed or illegal URI.
- //
- throw new URISyntaxException(uri.toString(),
- "Expected URI with a valid authority");
- }
-
- // The URI has a valid authority. Extract the container name. It is the
- // second component of the WASB URI authority.
- if (!authority.contains(WASB_AUTHORITY_DELIMITER)) {
- // The authority does not have a container name. Use the default container by
- // setting the container name to the default Azure root container.
- //
- return AZURE_ROOT_CONTAINER;
- }
-
- // Split off the container name and the authority.
- String[] authorityParts = authority.split(WASB_AUTHORITY_DELIMITER, 2);
-
- // Because the string contains an '@' delimiter, a container must be
- // specified.
- if (authorityParts.length < 2 || "".equals(authorityParts[0])) {
- // Badly formed WASB authority since there is no container.
- final String errMsg = String
- .format(
- "URI '%s' has a malformed WASB authority, expected container name."
- + "Authority takes the form wasb://[<container name>@]<account name>",
- uri.toString());
- throw new IllegalArgumentException(errMsg);
- }
-
- // Set the container name from the first entry for the split parts of the
- // authority.
- return authorityParts[0];
- }
-
- /**
- * Get the appropriate return the appropriate scheme for communicating with
- * Azure depending on whether wasb or wasbs is specified in the target URI.
- *
- * return scheme - HTTPS or HTTP as appropriate.
- */
- private String getHTTPScheme() {
- String sessionScheme = sessionUri.getScheme();
- // Check if we're on a secure URI scheme: wasbs or the legacy asvs scheme.
- if (sessionScheme != null
- && (sessionScheme.equalsIgnoreCase("asvs")
- || sessionScheme.equalsIgnoreCase("wasbs"))) {
- return HTTPS_SCHEME;
- } else {
- // At this point the scheme should be either null or asv or wasb.
- // Intentionally I'm not going to validate it though since I don't feel
- // it's this method's job to ensure a valid URI scheme for this file
- // system.
- return HTTP_SCHEME;
- }
- }
-
- /**
- * Set the configuration parameters for this client storage session with
- * Azure.
- *
- * @throws AzureException
- */
- private void configureAzureStorageSession() throws AzureException {
-
- // Assertion: Target session URI already should have been captured.
- if (sessionUri == null) {
- throw new AssertionError(
- "Expected a non-null session URI when configuring storage session");
- }
-
- // Assertion: A client session already should have been established with
- // Azure.
- if (storageInteractionLayer == null) {
- throw new AssertionError(String.format(
- "Cannot configure storage session for URI '%s' "
- + "if storage session has not been established.",
- sessionUri.toString()));
- }
-
- // Determine whether or not reads are allowed concurrent with OOB writes.
- tolerateOobAppends = sessionConfiguration.getBoolean(
- KEY_READ_TOLERATE_CONCURRENT_APPEND,
- DEFAULT_READ_TOLERATE_CONCURRENT_APPEND);
-
- // Retrieve configuration for the minimum stream read and write block size.
- //
- this.downloadBlockSizeBytes = sessionConfiguration.getInt(
- KEY_STREAM_MIN_READ_SIZE, DEFAULT_DOWNLOAD_BLOCK_SIZE);
- this.uploadBlockSizeBytes = sessionConfiguration.getInt(
- KEY_WRITE_BLOCK_SIZE, DEFAULT_UPLOAD_BLOCK_SIZE);
- this.hadoopBlockSize = sessionConfiguration.getLong(
- HADOOP_BLOCK_SIZE_PROPERTY_NAME, DEFAULT_HADOOP_BLOCK_SIZE);
-
- this.inputStreamVersion = sessionConfiguration.getInt(
- KEY_INPUT_STREAM_VERSION, DEFAULT_INPUT_STREAM_VERSION);
-
- // The job may want to specify a timeout to use when engaging the
- // storage service. The default is currently 90 seconds. It may
- // be necessary to increase this value for long latencies in larger
- // jobs. If the timeout specified is greater than zero seconds use
- // it, otherwise use the default service client timeout.
- int storageConnectionTimeout = sessionConfiguration.getInt(
- KEY_STORAGE_CONNECTION_TIMEOUT, 0);
-
- if (0 < storageConnectionTimeout) {
- storageInteractionLayer.setTimeoutInMs(storageConnectionTimeout * 1000);
- }
-
- // Set the concurrency values equal to the that specified in the
- // configuration file. If it does not exist, set it to the default
- // value calculated as double the number of CPU cores on the client
- // machine. The concurrency value is minimum of double the cores and
- // the read/write property.
- int cpuCores = 2 * Runtime.getRuntime().availableProcessors();
-
- concurrentWrites = sessionConfiguration.getInt(
- KEY_CONCURRENT_CONNECTION_VALUE_OUT,
- Math.min(cpuCores, DEFAULT_CONCURRENT_WRITES));
-
- // Set up the exponential retry policy.
- //
- minBackoff = sessionConfiguration.getInt(
- KEY_MIN_BACKOFF_INTERVAL, DEFAULT_MIN_BACKOFF_INTERVAL);
-
- maxBackoff = sessionConfiguration.getInt(
- KEY_MAX_BACKOFF_INTERVAL, DEFAULT_MAX_BACKOFF_INTERVAL);
-
- deltaBackoff = sessionConfiguration.getInt(
- KEY_BACKOFF_INTERVAL, DEFAULT_BACKOFF_INTERVAL);
-
- maxRetries = sessionConfiguration.getInt(
- KEY_MAX_IO_RETRIES, DEFAULT_MAX_RETRY_ATTEMPTS);
-
- storageInteractionLayer.setRetryPolicyFactory(
- new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries));
-
-
- // read the self-throttling config.
- selfThrottlingEnabled = sessionConfiguration.getBoolean(
- KEY_SELF_THROTTLE_ENABLE, DEFAULT_SELF_THROTTLE_ENABLE);
-
- selfThrottlingReadFactor = sessionConfiguration.getFloat(
- KEY_SELF_THROTTLE_READ_FACTOR, DEFAULT_SELF_THROTTLE_READ_FACTOR);
-
- selfThrottlingWriteFactor = sessionConfiguration.getFloat(
- KEY_SELF_THROTTLE_WRITE_FACTOR, DEFAULT_SELF_THROTTLE_WRITE_FACTOR);
-
- if (!selfThrottlingEnabled) {
- autoThrottlingEnabled = sessionConfiguration.getBoolean(
- KEY_AUTO_THROTTLE_ENABLE,
- DEFAULT_AUTO_THROTTLE_ENABLE);
- if (autoThrottlingEnabled) {
- ClientThrottlingIntercept.initializeSingleton();
- }
- } else {
- // cannot enable both self-throttling and client-throttling
- autoThrottlingEnabled = false;
- }
-
- OperationContext.setLoggingEnabledByDefault(sessionConfiguration.
- getBoolean(KEY_ENABLE_STORAGE_CLIENT_LOGGING, false));
-
- LOG.debug(
- "AzureNativeFileSystemStore init. Settings={},{},{},{{},{},{},{}},{{},{},{}}",
- concurrentWrites, tolerateOobAppends,
- ((storageConnectionTimeout > 0) ? storageConnectionTimeout
- : STORAGE_CONNECTION_TIMEOUT_DEFAULT), minBackoff,
- deltaBackoff, maxBackoff, maxRetries, selfThrottlingEnabled,
- selfThrottlingReadFactor, selfThrottlingWriteFactor);
- }
-
- /**
- * Connect to Azure storage using anonymous credentials.
- *
- * @param uri
- * - URI to target blob (R/O access to public blob)
- *
- * @throws StorageException
- * raised on errors communicating with Azure storage.
- * @throws IOException
- * raised on errors performing I/O or setting up the session.
- * @throws URISyntaxException
- * raised on creating mal-formed URI's.
- */
- private void connectUsingAnonymousCredentials(final URI uri)
- throws StorageException, IOException, URISyntaxException {
- // Use an HTTP scheme since the URI specifies a publicly accessible
- // container. Explicitly create a storage URI corresponding to the URI
- // parameter for use in creating the service client.
- String accountName = getAccountFromAuthority(uri);
- URI storageUri = new URI(getHTTPScheme() + ":" + PATH_DELIMITER
- + PATH_DELIMITER + accountName);
-
- // Create the service client with anonymous credentials.
- String containerName = getContainerFromAuthority(uri);
- storageInteractionLayer.createBlobClient(storageUri);
- suppressRetryPolicyInClientIfNeeded();
-
- // Capture the container reference.
- container = storageInteractionLayer.getContainerReference(containerName);
- rootDirectory = container.getDirectoryReference("");
-
- // Check for container existence, and our ability to access it.
- boolean canAccess;
- try {
- canAccess = container.exists(getInstrumentedContext());
- } catch (StorageException ex) {
- LOG.error("Service returned StorageException when checking existence "
- + "of container {} in account {}", containerName, accountName, ex);
- canAccess = false;
- }
- if (!canAccess) {
- throw new AzureException(String.format(NO_ACCESS_TO_CONTAINER_MSG,
- accountName, containerName));
- }
-
- // Accessing the storage server unauthenticated using
- // anonymous credentials.
- isAnonymousCredentials = true;
- }
-
- private void connectUsingCredentials(String accountName,
- StorageCredentials credentials, String containerName)
- throws URISyntaxException, StorageException, AzureException {
-
- URI blobEndPoint;
- if (isStorageEmulatorAccount(accountName)) {
- isStorageEmulator = true;
- CloudStorageAccount account =
- CloudStorageAccount.getDevelopmentStorageAccount();
- storageInteractionLayer.createBlobClient(account);
- } else {
- blobEndPoint = new URI(getHTTPScheme() + "://" + accountName);
- storageInteractionLayer.createBlobClient(blobEndPoint, credentials);
- }
- suppressRetryPolicyInClientIfNeeded();
-
- // Capture the container reference for debugging purposes.
- container = storageInteractionLayer.getContainerReference(containerName);
- rootDirectory = container.getDirectoryReference("");
-
- // Can only create container if using account key credentials
- canCreateOrModifyContainer = credentials instanceof StorageCredentialsAccountAndKey;
- }
-
- /**
- * Method to set up the Storage Interaction layer in Secure mode.
- * @param accountName - Storage account provided in the initializer
- * @param containerName - Container name provided in the initializer
- * @param sessionUri - URI provided in the initializer
- */
- private void connectToAzureStorageInSecureMode(String accountName,
- String containerName, URI sessionUri)
- throws AzureException, StorageException, URISyntaxException {
-
- LOG.debug("Connecting to Azure storage in Secure Mode");
- // Assertion: storageInteractionLayer instance has to be a SecureStorageInterfaceImpl
- if (!(this.storageInteractionLayer instanceof SecureStorageInterfaceImpl)) {
- throw new AssertionError("connectToAzureStorageInSecureMode() should be called only"
- + " for SecureStorageInterfaceImpl instances");
- }
-
- ((SecureStorageInterfaceImpl) this.storageInteractionLayer).
- setStorageAccountName(accountName);
- connectingUsingSAS = true;
- container = storageInteractionLayer.getContainerReference(containerName);
- rootDirectory = container.getDirectoryReference("");
-
- canCreateOrModifyContainer = true;
- }
-
- /**
- * Connect to Azure storage using account key credentials.
- */
- private void connectUsingConnectionStringCredentials(
- final String accountName, final String containerName,
- final String accountKey) throws InvalidKeyException, StorageException,
- IOException, URISyntaxException {
- // If the account name is "acc.blob.core.windows.net", then the
- // rawAccountName is just "acc"
- String rawAccountName = accountName.split("\\.")[0];
- StorageCredentials credentials = new StorageCredentialsAccountAndKey(
- rawAccountName, accountKey);
- connectUsingCredentials(accountName, credentials, containerName);
- }
-
- /**
- * Connect to Azure storage using shared access signature credentials.
- */
- private void connectUsingSASCredentials(final String accountName,
- final String containerName, final String sas) throws InvalidKeyException,
- StorageException, IOException, URISyntaxException {
- StorageCredentials credentials = new StorageCredentialsSharedAccessSignature(
- sas);
- connectingUsingSAS = true;
- connectUsingCredentials(accountName, credentials, containerName);
- }
-
- private boolean isStorageEmulatorAccount(final String accountName) {
- return accountName.equalsIgnoreCase(sessionConfiguration.get(
- STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME,
- DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME));
- }
-
- @VisibleForTesting
- public static String getAccountKeyFromConfiguration(String accountName,
- Configuration conf) throws KeyProviderException {
- String key = null;
- String keyProviderClass = conf.get(KEY_ACCOUNT_KEYPROVIDER_PREFIX
- + accountName);
- KeyProvider keyProvider = null;
-
- if (keyProviderClass == null) {
- // No key provider was provided so use the provided key as is.
- keyProvider = new SimpleKeyProvider();
- } else {
- // create an instance of the key provider class and verify it
- // implements KeyProvider
- Object keyProviderObject = null;
- try {
- Class<?> clazz = conf.getClassByName(keyProviderClass);
- keyProviderObject = clazz.newInstance();
- } catch (Exception e) {
- throw new KeyProviderException("Unable to load key provider class.", e);
- }
- if (!(keyProviderObject instanceof KeyProvider)) {
- throw new KeyProviderException(keyProviderClass
- + " specified in config is not a valid KeyProvider class.");
- }
- keyProvider = (KeyProvider) keyProviderObject;
- }
- key = keyProvider.getStorageAccountKey(accountName, conf);
-
- return key;
- }
-
- /**
- * Establish a session with Azure blob storage based on the target URI. The
- * method determines whether or not the URI target contains an explicit
- * account or an implicit default cluster-wide account.
- *
- * @throws AzureException
- * @throws IOException
- */
- private void createAzureStorageSession()
- throws AzureException, IOException {
-
- // Make sure this object was properly initialized with references to
- // the sessionUri and sessionConfiguration.
- if (null == sessionUri || null == sessionConfiguration) {
- throw new AzureException("Filesystem object not initialized properly."
- + "Unable to start session with Azure Storage server.");
- }
-
- // File system object initialized, attempt to establish a session
- // with the Azure storage service for the target URI string.
- try {
- // Inspect the URI authority to determine the account and use the account
- // to start an Azure blob client session using an account key for the
- // the account or anonymously.
- // For all URI's do the following checks in order:
- // 1. Validate that <account> can be used with the current Hadoop
- // cluster by checking it exists in the list of configured accounts
- // for the cluster.
- // 2. Look up the AccountKey in the list of configured accounts for the
- // cluster.
- // 3. If there is no AccountKey, assume anonymous public blob access
- // when accessing the blob.
- //
- // If the URI does not specify a container use the default root container
- // under the account name.
-
- // Assertion: Container name on the session Uri should be non-null.
- if (getContainerFromAuthority(sessionUri) == null) {
- throw new AssertionError(String.format(
- "Non-null container expected from session URI: %s.",
- sessionUri.toString()));
- }
-
- // Get the account name.
- String accountName = getAccountFromAuthority(sessionUri);
- if (null == accountName) {
- // Account name is not specified as part of the URI. Throw indicating
- // an invalid account name.
- final String errMsg = String.format(
- "Cannot load WASB file system account name not"
- + " specified in URI: %s.", sessionUri.toString());
- throw new AzureException(errMsg);
- }
-
- instrumentation.setAccountName(accountName);
- String containerName = getContainerFromAuthority(sessionUri);
- instrumentation.setContainerName(containerName);
-
- // Check whether this is a storage emulator account.
- if (isStorageEmulatorAccount(accountName)) {
- // It is an emulator account, connect to it with no credentials.
- connectUsingCredentials(accountName, null, containerName);
- return;
- }
-
- // If the securemode flag is set, WASB uses SecureStorageInterfaceImpl instance
- // to communicate with Azure storage. In SecureStorageInterfaceImpl SAS keys
- // are used to communicate with Azure storage, so connectToAzureStorageInSecureMode
- // instantiates the default container using a SAS Key.
- if (useSecureMode) {
- connectToAzureStorageInSecureMode(accountName, containerName, sessionUri);
- return;
- }
-
- // Check whether we have a shared access signature for that container.
- String propertyValue = sessionConfiguration.get(KEY_ACCOUNT_SAS_PREFIX
- + containerName + "." + accountName);
- if (propertyValue != null) {
- // SAS was found. Connect using that.
- connectUsingSASCredentials(accountName, containerName, propertyValue);
- return;
- }
-
- // Check whether the account is configured with an account key.
- propertyValue = getAccountKeyFromConfiguration(accountName,
- sessionConfiguration);
- if (StringUtils.isNotEmpty(propertyValue)) {
- // Account key was found.
- // Create the Azure storage session using the account key and container.
- connectUsingConnectionStringCredentials(
- getAccountFromAuthority(sessionUri),
- getContainerFromAuthority(sessionUri), propertyValue);
- } else {
- LOG.debug("The account access key is not configured for {}. "
- + "Now try anonymous access.", sessionUri);
- connectUsingAnonymousCredentials(sessionUri);
- }
- } catch (Exception e) {
- // Caught exception while attempting to initialize the Azure File
- // System store, re-throw the exception.
- throw new AzureException(e);
- }
- }
-
- private enum ContainerState {
- /**
- * We haven't checked the container state yet.
- */
- Unknown,
- /**
- * We checked and the container doesn't exist.
- */
- DoesntExist,
- /**
- * The container exists and doesn't have an WASB version stamp on it.
- */
- ExistsNoVersion,
- /**
- * The container exists and has an unsupported WASB version stamped on it.
- */
- ExistsAtWrongVersion,
- /**
- * The container exists and has the proper WASB version stamped on it.
- */
- ExistsAtRightVersion
- }
-
- private enum ContainerAccessType {
- /**
- * We're accessing the container for a pure read operation, e.g. read a
- * file.
- */
- PureRead,
- /**
- * We're accessing the container purely to write something, e.g. write a
- * file.
- */
- PureWrite,
- /**
- * We're accessing the container to read something then write, e.g. rename a
- * file.
- */
- ReadThenWrite
- }
-
- /**
- * Trims a suffix/prefix from the given string. For example if
- * s is given as "/xy" and toTrim is "/", this method returns "xy"
- */
- private static String trim(String s, String toTrim) {
- return StringUtils.removeEnd(StringUtils.removeStart(s, toTrim),
- toTrim);
- }
-
- /**
- * Checks if the given rawDir belongs to this account/container, and
- * if so returns the canonicalized path for it. Otherwise return null.
- */
- private String verifyAndConvertToStandardFormat(String rawDir) throws URISyntaxException {
- URI asUri = new URI(rawDir);
- if (asUri.getAuthority() == null
- || asUri.getAuthority().toLowerCase(Locale.ENGLISH).equalsIgnoreCase(
- sessionUri.getAuthority().toLowerCase(Locale.ENGLISH))) {
- // Applies to me.
- return trim(asUri.getPath(), "/");
- } else {
- // Doen't apply to me.
- return null;
- }
- }
-
- /**
- * Take a comma-separated list of directories from a configuration variable
- * and transform it to a set of directories.
- */
- private Set<String> getDirectorySet(final String configVar)
- throws AzureException {
- String[] rawDirs = sessionConfiguration.getStrings(configVar, new String[0]);
- Set<String> directorySet = new HashSet<String>();
- for (String currentDir : rawDirs) {
- String myDir;
- try {
- myDir = verifyAndConvertToStandardFormat(currentDir.trim());
- } catch (URISyntaxException ex) {
- throw new AzureException(String.format(
- "The directory %s specified in the configuration entry %s is not"
- + " a valid URI.",
- currentDir, configVar));
- }
- if (myDir != null) {
- directorySet.add(myDir);
- }
- }
- return directorySet;
- }
-
- /**
- * Checks if the given key in Azure Storage should be stored as a page
- * blob instead of block blob.
- */
- public boolean isPageBlobKey(String key) {
- return isKeyForDirectorySet(key, pageBlobDirs);
- }
-
- /**
- * Checks if the given key in Azure Storage should be stored as a block blobs
- * with compaction enabled instead of normal block blob.
- *
- * @param key blob name
- * @return true, if the file is in directory with block compaction enabled.
- */
- public boolean isBlockBlobWithCompactionKey(String key) {
- return isKeyForDirectorySet(key, blockBlobWithCompationDirs);
- }
-
- /**
- * Checks if the given key in Azure storage should have synchronized
- * atomic folder rename createNonRecursive implemented.
- */
- @Override
- public boolean isAtomicRenameKey(String key) {
- return isKeyForDirectorySet(key, atomicRenameDirs);
- }
-
- public boolean isKeyForDirectorySet(String key, Set<String> dirSet) {
- String defaultFS = FileSystem.getDefaultUri(sessionConfiguration).toString();
- for (String dir : dirSet) {
- if (dir.isEmpty()) {
- // dir is root
- return true;
- }
-
- if (matchAsteriskPattern(key, dir)) {
- return true;
- }
-
- // Allow for blob directories with paths relative to the default file
- // system.
- //
- try {
- URI uriPageBlobDir = new URI(dir);
- if (null == uriPageBlobDir.getAuthority()) {
- // Concatenate the default file system prefix with the relative
- // page blob directory path.
- //
- String dirWithPrefix = trim(defaultFS, "/") + "/" + dir;
- if (matchAsteriskPattern(key, dirWithPrefix)) {
- return true;
- }
- }
- } catch (URISyntaxException e) {
- LOG.info("URI syntax error creating URI for {}", dir);
- }
- }
- return false;
- }
-
- private boolean matchAsteriskPattern(String pathName, String pattern) {
- if (pathName == null || pathName.length() == 0) {
- return false;
- }
-
- int pathIndex = 0;
- int patternIndex = 0;
-
- while (pathIndex < pathName.length() && patternIndex < pattern.length()) {
- char charToMatch = pattern.charAt(patternIndex);
-
- // normal char:
- if (charToMatch != ASTERISK_SYMBOL) {
- if (charToMatch != pathName.charAt(pathIndex)) {
- return false;
- }
- pathIndex++;
- patternIndex++;
- continue;
- }
-
- // ASTERISK_SYMBOL
- // 1. * is used in path name: *a/b,a*/b, a/*b, a/b*
- if (patternIndex > 0 && pattern.charAt(patternIndex - 1) != Path.SEPARATOR_CHAR
- || patternIndex + 1 < pattern.length() && pattern.charAt(patternIndex + 1) != Path.SEPARATOR_CHAR) {
- if (ASTERISK_SYMBOL != pathName.charAt(pathIndex)) {
- return false;
- }
-
- pathIndex++;
- patternIndex++;
- continue;
- }
-
- // 2. * is used as wildcard: */a, a/*/b, a/*
- patternIndex++;
- // find next path separator
- while (pathIndex < pathName.length() && pathName.charAt(pathIndex) != Path.SEPARATOR_CHAR) {
- pathIndex++;
- }
- }
-
- // Ensure it is not a file/dir which shares same prefix as pattern
- // Eg: pattern: /A/B, pathName: /A/BBB should not match
- return patternIndex == pattern.length()
- && (pathIndex == pathName.length() || pathName.charAt(pathIndex) == Path.SEPARATOR_CHAR);
- }
-
- /**
- * Returns the file block size. This is a fake value used for integration
- * of the Azure store with Hadoop.
- */
- @Override
- public long getHadoopBlockSize() {
- return hadoopBlockSize;
- }
-
- /**
- * This should be called from any method that does any modifications to the
- * underlying container: it makes sure to put the WASB current version in the
- * container's metadata if it's not already there.
- */
- private ContainerState checkContainer(ContainerAccessType accessType)
- throws StorageException, AzureException {
- synchronized (containerStateLock) {
- if (isOkContainerState(accessType)) {
- return currentKnownContainerState;
- }
- if (currentKnownContainerState == ContainerState.ExistsAtWrongVersion) {
- String containerVersion = retrieveVersionAttribute(container);
- throw wrongVersionException(containerVersion);
- }
- // This means I didn't check it before or it didn't exist or
- // we need to stamp the version. Since things may have changed by
- // other machines since then, do the check again and don't depend
- // on past information.
-
- // Sanity check: we don't expect this at this point.
- if (currentKnownContainerState == ContainerState.ExistsAtRightVersion) {
- throw new AssertionError("Unexpected state: "
- + currentKnownContainerState);
- }
-
- // Download the attributes - doubles as an existence check with just
- // one service call
- try {
- container.downloadAttributes(getInstrumentedContext());
- currentKnownContainerState = ContainerState.Unknown;
- } catch (StorageException ex) {
- if (StorageErrorCodeStrings.CONTAINER_NOT_FOUND.toString()
- .equals(ex.getErrorCode())) {
- currentKnownContainerState = ContainerState.DoesntExist;
- } else {
- throw ex;
- }
- }
-
- if (currentKnownContainerState == ContainerState.DoesntExist) {
- // If the container doesn't exist and we intend to write to it,
- // create it now.
- if (needToCreateContainer(accessType)) {
- storeVersionAttribute(container);
- container.create(getInstrumentedContext());
- currentKnownContainerState = ContainerState.ExistsAtRightVersion;
- }
- } else {
- // The container exists, check the version.
- String containerVersion = retrieveVersionAttribute(container);
- if (containerVersion != null) {
- if (containerVersion.equals(FIRST_WASB_VERSION)) {
- // It's the version from when WASB was called ASV, just
- // fix the version attribute if needed and proceed.
- // We should be good otherwise.
- if (needToStampVersion(accessType)) {
- storeVersionAttribute(container);
- container.uploadMetadata(getInstrumentedContext());
- }
- } else if (!containerVersion.equals(CURRENT_WASB_VERSION)) {
- // Don't know this version - throw.
- currentKnownContainerState = ContainerState.ExistsAtWrongVersion;
- throw wrongVersionException(containerVersion);
- } else {
- // It's our correct version.
- currentKnownContainerState = ContainerState.ExistsAtRightVersion;
- }
- } else {
- // No version info exists.
- currentKnownContainerState = ContainerState.ExistsNoVersion;
- if (needToStampVersion(accessType)) {
- // Need to stamp the version
- storeVersionAttribute(container);
- container.uploadMetadata(getInstrumentedContext());
- currentKnownContainerState = ContainerState.ExistsAtRightVersion;
- }
- }
- }
- return currentKnownContainerState;
- }
- }
-
- private AzureException wrongVersionException(String containerVersion) {
- return new AzureException("The container " + container.getName()
- + " is at an unsupported version: " + containerVersion
- + ". Current supported version: " + FIRST_WASB_VERSION);
- }
-
- private boolean needToStampVersion(ContainerAccessType accessType) {
- // We need to stamp the version on the container any time we write to
- // it and we have the correct credentials to be able to write container
- // metadata.
- return accessType != ContainerAccessType.PureRead
- && canCreateOrModifyContainer;
- }
-
- private static boolean needToCreateContainer(ContainerAccessType accessType) {
- // We need to pro-actively create the container (if it doesn't exist) if
- // we're doing a pure write. No need to create it for pure read or read-
- // then-write access.
- return accessType == ContainerAccessType.PureWrite;
- }
-
- // Determines whether we have to pull the container information again
- // or we can work based off what we already have.
- private boolean isOkContainerState(ContainerAccessType accessType) {
- switch (currentKnownContainerState) {
- case Unknown:
- // When using SAS, we can't discover container attributes
- // so just live with Unknown state and fail later if it
- // doesn't exist.
- return connectingUsingSAS;
- case DoesntExist:
- return false; // the container could have been created
- case ExistsAtRightVersion:
- return true; // fine to optimize
- case ExistsAtWrongVersion:
- return false;
- case ExistsNoVersion:
- // If there's no version, it's OK if we don't need to stamp the version
- // or we can't anyway even if we wanted to.
- return !needToStampVersion(accessType);
- default:
- throw new AssertionError("Unknown access type: " + accessType);
- }
- }
-
- private boolean getUseTransactionalContentMD5() {
- return sessionConfiguration.getBoolean(KEY_CHECK_BLOCK_MD5, true);
- }
-
- private BlobRequestOptions getUploadOptions() {
- BlobRequestOptions options = new BlobRequestOptions();
- options.setStoreBlobContentMD5(sessionConfiguration.getBoolean(
- KEY_STORE_BLOB_MD5, false));
- options.setUseTransactionalContentMD5(getUseTransactionalContentMD5());
- options.setConcurrentRequestCount(concurrentWrites);
-
- options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff,
- deltaBackoff, maxBackoff, maxRetries));
-
- return options;
- }
-
- private BlobRequestOptions getDownloadOptions() {
- BlobRequestOptions options = new BlobRequestOptions();
- options.setRetryPolicyFactory(
- new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries));
- options.setUseTransactionalContentMD5(getUseTransactionalContentMD5());
- return options;
- }
-
- @Override
- public DataOutputStream storefile(String keyEncoded,
- PermissionStatus permissionStatus,
- String key)
- throws AzureException {
- try {
-
- // Check if a session exists, if not create a session with the
- // Azure storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AzureException(errMsg);
- }
-
- // Check if there is an authenticated account associated with the
- // file this instance of the WASB file system. If not the file system
- // has not been authenticated and all access is anonymous.
- if (!isAuthenticatedAccess()) {
- // Preemptively raise an exception indicating no uploads are
- // allowed to anonymous accounts.
- throw new AzureException(new IOException(
- "Uploads to public accounts using anonymous "
- + "access is prohibited."));
- }
-
- checkContainer(ContainerAccessType.PureWrite);
-
- /**
- * Note: Windows Azure Blob Storage does not allow the creation of arbitrary directory
- * paths under the default $root directory. This is by design to eliminate
- * ambiguity in specifying a implicit blob address. A blob in the $root conatiner
- * cannot include a / in its name and must be careful not to include a trailing
- * '/' when referencing blobs in the $root container.
- * A '/; in the $root container permits ambiguous blob names as in the following
- * example involving two containers $root and mycontainer:
- * http://myaccount.blob.core.windows.net/$root
- * http://myaccount.blob.core.windows.net/mycontainer
- * If the URL "mycontainer/somefile.txt were allowed in $root then the URL:
- * http://myaccount.blob.core.windows.net/mycontainer/myblob.txt
- * could mean either:
- * (1) container=mycontainer; blob=myblob.txt
- * (2) container=$root; blob=mycontainer/myblob.txt
- *
- * To avoid this type of ambiguity the Azure blob storage prevents
- * arbitrary path under $root. For a simple and more consistent user
- * experience it was decided to eliminate the opportunity for creating
- * such paths by making the $root container read-only under WASB.
- */
-
- // Check that no attempt is made to write to blobs on default
- // $root containers.
- if (AZURE_ROOT_CONTAINER.equals(getContainerFromAuthority(sessionUri))) {
- // Azure containers are restricted to non-root containers.
- final String errMsg = String.format(
- "Writes to '%s' container for URI '%s' are prohibited, "
- + "only updates on non-root containers permitted.",
- AZURE_ROOT_CONTAINER, sessionUri.toString());
- throw new AzureException(errMsg);
- }
-
- // Get the blob reference from the store's container and
- // return it.
- CloudBlobWrapper blob = getBlobReference(keyEncoded);
- storePermissionStatus(blob, permissionStatus);
-
- // Create the output stream for the Azure blob.
- //
- OutputStream outputStream;
-
- if (isBlockBlobWithCompactionKey(key)) {
- BlockBlobAppendStream blockBlobOutputStream = new BlockBlobAppendStream(
- (CloudBlockBlobWrapper) blob,
- keyEncoded,
- this.uploadBlockSizeBytes,
- true,
- getInstrumentedContext());
-
- outputStream = blockBlobOutputStream;
- } else {
- outputStream = openOutputStream(blob);
- }
-
- DataOutputStream dataOutStream = new SyncableDataOutputStream(outputStream);
- return dataOutStream;
- } catch (Exception e) {
- // Caught exception while attempting to open the blob output stream.
- // Re-throw as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- /**
- * Opens a new output stream to the given blob (page or block blob)
- * to populate it from scratch with data.
- */
- private OutputStream openOutputStream(final CloudBlobWrapper blob)
- throws StorageException {
- if (blob instanceof CloudPageBlobWrapper){
- return new PageBlobOutputStream(
- (CloudPageBlobWrapper) blob, getInstrumentedContext(), sessionConfiguration);
- } else {
-
- // Handle both ClouldBlockBlobWrapperImpl and (only for the test code path)
- // MockCloudBlockBlobWrapper.
- return ((CloudBlockBlobWrapper) blob).openOutputStream(getUploadOptions(),
- getInstrumentedContext());
- }
- }
-
- /**
- * Opens a new input stream for the given blob (page or block blob)
- * to read its data.
- */
- private InputStream openInputStream(CloudBlobWrapper blob,
- Optional<Configuration> options) throws StorageException, IOException {
- if (blob instanceof CloudBlockBlobWrapper) {
- LOG.debug("Using stream seek algorithm {}", inputStreamVersion);
- switch(inputStreamVersion) {
- case 1:
- return blob.openInputStream(getDownloadOptions(),
- getInstrumentedContext(isConcurrentOOBAppendAllowed()));
- case 2:
- boolean bufferedPreadDisabled = options.map(c -> c
- .getBoolean(FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE, false))
- .orElse(false);
- return new BlockBlobInputStream((CloudBlockBlobWrapper) blob,
- getDownloadOptions(),
- getInstrumentedContext(isConcurrentOOBAppendAllowed()),
- bufferedPreadDisabled);
- default:
- throw new IOException("Unknown seek algorithm: " + inputStreamVersion);
- }
- } else {
- return new PageBlobInputStream(
- (CloudPageBlobWrapper) blob, getInstrumentedContext(
- isConcurrentOOBAppendAllowed()));
- }
- }
-
- /**
- * Default permission to use when no permission metadata is found.
- *
- * @return The default permission to use.
- */
- private static PermissionStatus defaultPermissionNoBlobMetadata() {
- return new PermissionStatus("", "", FsPermission.getDefault());
- }
-
- private static void storeMetadataAttribute(CloudBlobWrapper blob,
- String key, String value) {
- HashMap<String, String> metadata = blob.getMetadata();
- if (null == metadata) {
- metadata = new HashMap<String, String>();
- }
- metadata.put(key, value);
- blob.setMetadata(metadata);
- }
-
- private String getMetadataAttribute(HashMap<String, String> metadata,
- String... keyAlternatives) {
- if (null == metadata) {
- return null;
- }
- for (String key : keyAlternatives) {
- if (metadataKeyCaseSensitive) {
- if (metadata.containsKey(key)) {
- return metadata.get(key);
- }
- } else {
- // See HADOOP-17643 for details on why this case insensitive metadata
- // checks been added
- for (Entry<String, String> entry : metadata.entrySet()) {
- if (key.equalsIgnoreCase(entry.getKey())) {
- return entry.getValue();
- }
- }
- }
- }
- return null;
- }
-
- private static void removeMetadataAttribute(CloudBlobWrapper blob,
- String key) {
- HashMap<String, String> metadata = blob.getMetadata();
- if (metadata != null) {
- metadata.remove(key);
- blob.setMetadata(metadata);
- }
- }
-
- private static void storePermissionStatus(CloudBlobWrapper blob,
- PermissionStatus permissionStatus) {
- storeMetadataAttribute(blob, PERMISSION_METADATA_KEY,
- PERMISSION_JSON_SERIALIZER.toJSON(permissionStatus));
- // Remove the old metadata key if present
- removeMetadataAttribute(blob, OLD_PERMISSION_METADATA_KEY);
- }
-
- private PermissionStatus getPermissionStatus(CloudBlobWrapper blob) {
- String permissionMetadataValue = getMetadataAttribute(blob.getMetadata(),
- PERMISSION_METADATA_KEY, OLD_PERMISSION_METADATA_KEY);
- if (permissionMetadataValue != null) {
- return PermissionStatusJsonSerializer.fromJSONString(
- permissionMetadataValue);
- } else {
- return defaultPermissionNoBlobMetadata();
- }
- }
-
- private static void storeFolderAttribute(CloudBlobWrapper blob) {
- storeMetadataAttribute(blob, IS_FOLDER_METADATA_KEY, "true");
- // Remove the old metadata key if present
- removeMetadataAttribute(blob, OLD_IS_FOLDER_METADATA_KEY);
- }
-
- private static String encodeMetadataAttribute(String value) throws UnsupportedEncodingException {
- // We have to URL encode the attribute as it could
- // have URI special characters which unless encoded will result
- // in 403 errors from the server. This is due to metadata properties
- // being sent in the HTTP header of the request which is in turn used
- // on the server side to authorize the request.
- return value == null ? null : URLEncoder.encode(value, METADATA_ENCODING.name());
- }
-
- private static String decodeMetadataAttribute(String encoded) throws UnsupportedEncodingException {
- return encoded == null ? null : URLDecoder.decode(encoded, METADATA_ENCODING.name());
- }
-
- private static String ensureValidAttributeName(String attribute) {
- // Attribute names must be valid C# identifiers so we have to
- // convert the namespace dots (e.g. "user.something") in the
- // attribute names. Using underscores here to be consistent with
- // the constant metadata keys defined earlier in the file
- return attribute.replace('.', '_');
- }
-
- private static void storeLinkAttribute(CloudBlobWrapper blob,
- String linkTarget) throws UnsupportedEncodingException {
- String encodedLinkTarget = encodeMetadataAttribute(linkTarget);
- storeMetadataAttribute(blob,
- LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
- encodedLinkTarget);
- // Remove the old metadata key if present
- removeMetadataAttribute(blob,
- OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
- }
-
- private String getLinkAttributeValue(CloudBlobWrapper blob)
- throws UnsupportedEncodingException {
- String encodedLinkTarget = getMetadataAttribute(blob.getMetadata(),
- LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
- OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
- return decodeMetadataAttribute(encodedLinkTarget);
- }
-
- private boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
- HashMap<String, String> metadata = blob.getMetadata();
- if (null != metadata) {
- if (metadataKeyCaseSensitive) {
- return metadata.containsKey(IS_FOLDER_METADATA_KEY)
- || metadata.containsKey(OLD_IS_FOLDER_METADATA_KEY);
- } else {
- // See HADOOP-17643 for details on why this case insensitive metadata
- // checks been added
- for (String key : metadata.keySet()) {
- if (key.equalsIgnoreCase(IS_FOLDER_METADATA_KEY)
- || key.equalsIgnoreCase(OLD_IS_FOLDER_METADATA_KEY)) {
- return true;
- }
- }
- }
- }
- return false;
- }
-
- private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
- HashMap<String, String> metadata = container.getMetadata();
- if (null == metadata) {
- metadata = new HashMap<String, String>();
- }
- metadata.put(VERSION_METADATA_KEY, CURRENT_WASB_VERSION);
- if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) {
- metadata.remove(OLD_VERSION_METADATA_KEY);
- }
- container.setMetadata(metadata);
- }
-
- private String retrieveVersionAttribute(CloudBlobContainerWrapper container) {
- return getMetadataAttribute(container.getMetadata(), VERSION_METADATA_KEY,
- OLD_VERSION_METADATA_KEY);
- }
-
- @Override
- public void storeEmptyFolder(String key, PermissionStatus permissionStatus)
- throws AzureException {
-
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- // Check if there is an authenticated account associated with the file
- // this instance of the WASB file system. If not the file system has not
- // been authenticated and all access is anonymous.
- if (!isAuthenticatedAccess()) {
- // Preemptively raise an exception indicating no uploads are
- // allowed to anonymous accounts.
- throw new AzureException(
- "Uploads to to public accounts using anonymous access is prohibited.");
- }
-
- try {
- checkContainer(ContainerAccessType.PureWrite);
-
- CloudBlobWrapper blob = getBlobReference(key);
- storePermissionStatus(blob, permissionStatus);
- storeFolderAttribute(blob);
- openOutputStream(blob).close();
- } catch (StorageException e) {
- // Caught exception while attempting upload. Re-throw as an Azure
- // storage exception.
- throw new AzureException(e);
- } catch (URISyntaxException e) {
- throw new AzureException(e);
- } catch (IOException e) {
- Throwable t = e.getCause();
- if (t instanceof StorageException) {
- StorageException se = (StorageException) t;
- // If we got this exception, the blob should have already been created
- if (!"LeaseIdMissing".equals(se.getErrorCode())) {
- throw new AzureException(e);
- }
- } else {
- throw new AzureException(e);
- }
- }
- }
-
- /**
- * Stores an empty blob that's linking to the temporary file where're we're
- * uploading the initial data.
- */
- @Override
- public void storeEmptyLinkFile(String key, String tempBlobKey,
- PermissionStatus permissionStatus) throws AzureException {
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
- // Check if there is an authenticated account associated with the file
- // this instance of the WASB file system. If not the file system has not
- // been authenticated and all access is anonymous.
- if (!isAuthenticatedAccess()) {
- // Preemptively raise an exception indicating no uploads are
- // allowed to anonymous accounts.
- throw new AzureException(
- "Uploads to to public accounts using anonymous access is prohibited.");
- }
-
- try {
- checkContainer(ContainerAccessType.PureWrite);
-
- CloudBlobWrapper blob = getBlobReference(key);
- storePermissionStatus(blob, permissionStatus);
- storeLinkAttribute(blob, tempBlobKey);
- openOutputStream(blob).close();
- } catch (Exception e) {
- // Caught exception while attempting upload. Re-throw as an Azure
- // storage exception.
- throw new AzureException(e);
- }
- }
-
- /**
- * If the blob with the given key exists and has a link in its metadata to a
- * temporary file (see storeEmptyLinkFile), this method returns the key to
- * that temporary file. Otherwise, returns null.
- */
- @Override
- public String getLinkInFileMetadata(String key) throws AzureException {
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- try {
- checkContainer(ContainerAccessType.PureRead);
-
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
- return getLinkAttributeValue(blob);
- } catch (Exception e) {
- // Caught exception while attempting download. Re-throw as an Azure
- // storage exception.
- throw new AzureException(e);
- }
- }
-
- /**
- * Private method to check for authenticated access.
- *
- * @ returns boolean -- true if access is credentialed and authenticated and
- * false otherwise.
- */
- private boolean isAuthenticatedAccess() throws AzureException {
-
- if (isAnonymousCredentials) {
- // Access to this storage account is unauthenticated.
- return false;
- }
- // Access is authenticated.
- return true;
- }
-
- /**
- * This private method uses the root directory or the original container to
- * list blobs under the directory or container depending on whether the
- * original file system object was constructed with a short- or long-form URI.
- * If the root directory is non-null the URI in the file constructor was in
- * the long form.
- *
- * @param includeMetadata
- * if set, the listed items will have their metadata populated
- * already.
- * @param useFlatBlobListing
- * if set the list is flat, otherwise it is hierarchical.
- *
- * @returns blobItems : iterable collection of blob items.
- * @throws URISyntaxException
- *
- */
- private Iterable<ListBlobItem> listRootBlobs(boolean includeMetadata,
- boolean useFlatBlobListing) throws StorageException, URISyntaxException {
- return rootDirectory.listBlobs(
- null,
- useFlatBlobListing,
- includeMetadata
- ? EnumSet.of(BlobListingDetails.METADATA)
- : EnumSet.noneOf(BlobListingDetails.class),
- null,
- getInstrumentedContext());
- }
-
- /**
- * This private method uses the root directory or the original container to
- * list blobs under the directory or container given a specified prefix for
- * the directory depending on whether the original file system object was
- * constructed with a short- or long-form URI. If the root directory is
- * non-null the URI in the file constructor was in the long form.
- *
- * @param aPrefix
- * : string name representing the prefix of containing blobs.
- * @param includeMetadata
- * if set, the listed items will have their metadata populated
- * already.
- * @param useFlatBlobListing
- * if set the list is flat, otherwise it is hierarchical.
- *
- * @returns blobItems : iterable collection of blob items.
- * @throws URISyntaxException
- *
- */
- private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean includeMetadata,
- boolean useFlatBlobListing) throws StorageException, URISyntaxException {
-
- Iterable<ListBlobItem> list = rootDirectory.listBlobs(aPrefix,
- useFlatBlobListing,
- includeMetadata
- ? EnumSet.of(BlobListingDetails.METADATA)
- : EnumSet.noneOf(BlobListingDetails.class),
- null,
- getInstrumentedContext());
- return list;
- }
-
- /**
- * This private method uses the root directory or the original container to
- * list blobs under the directory or container given a specified prefix for
- * the directory depending on whether the original file system object was
- * constructed with a short- or long-form URI. It also uses the specified flat
- * or hierarchical option, listing details options, request options, and
- * operation context.
- *
- * @param aPrefix
- * string name representing the prefix of containing blobs.
- * @param useFlatBlobListing
- * - the list is flat if true, or hierarchical otherwise.
- * @param listingDetails
- * - determine whether snapshots, metadata, committed/uncommitted
- * data
- * @param options
- * - object specifying additional options for the request. null =
- * default options
- * @param opContext
- * - context of the current operation
- * @returns blobItems : iterable collection of blob items.
- * @throws URISyntaxException
- *
- */
- private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean useFlatBlobListing,
- EnumSet<BlobListingDetails> listingDetails, BlobRequestOptions options,
- OperationContext opContext) throws StorageException, URISyntaxException {
-
- CloudBlobDirectoryWrapper directory = this.container.getDirectoryReference(aPrefix);
- return directory.listBlobs(
- null,
- useFlatBlobListing,
- listingDetails,
- options,
- opContext);
- }
-
- /**
- * This private method uses the root directory or the original container to
- * get the block blob reference depending on whether the original file system
- * object was constructed with a short- or long-form URI. If the root
- * directory is non-null the URI in the file constructor was in the long form.
- *
- * @param aKey
- * : a key used to query Azure for the block blob.
- * @returns blob : a reference to the Azure block blob corresponding to the
- * key.
- * @throws URISyntaxException
- *
- */
- private CloudBlobWrapper getBlobReference(String aKey)
- throws StorageException, URISyntaxException {
-
- CloudBlobWrapper blob = null;
- if (isPageBlobKey(aKey)) {
- blob = this.container.getPageBlobReference(aKey);
- } else {
- blob = this.container.getBlockBlobReference(aKey);
- blob.setStreamMinimumReadSizeInBytes(downloadBlockSizeBytes);
- blob.setWriteBlockSizeInBytes(uploadBlockSizeBytes);
- }
-
- return blob;
- }
-
- /**
- * This private method normalizes the key by stripping the container name from
- * the path and returns a path relative to the root directory of the
- * container.
- *
- * @param keyUri
- * - adjust this key to a path relative to the root directory
- *
- * @returns normKey
- */
- private String normalizeKey(URI keyUri) {
- String normKey;
-
- // Strip the container name from the path and return the path
- // relative to the root directory of the container.
- int parts = isStorageEmulator ? 4 : 3;
- normKey = keyUri.getPath().split("/", parts)[(parts - 1)];
-
- // Return the fixed key.
- return normKey;
- }
-
- /**
- * This private method normalizes the key by stripping the container name from
- * the path and returns a path relative to the root directory of the
- * container.
- *
- * @param blob
- * - adjust the key to this blob to a path relative to the root
- * directory
- *
- * @returns normKey
- */
- private String normalizeKey(CloudBlobWrapper blob) {
- return normalizeKey(blob.getUri());
- }
-
- /**
- * This private method normalizes the key by stripping the container name from
- * the path and returns a path relative to the root directory of the
- * container.
- *
- * @param directory
- * - adjust the key to this directory to a path relative to the root
- * directory
- *
- * @returns normKey
- */
- private String normalizeKey(CloudBlobDirectoryWrapper directory) {
- String dirKey = normalizeKey(directory.getUri());
- // Strip the last delimiter
- if (dirKey.endsWith(PATH_DELIMITER)) {
- dirKey = dirKey.substring(0, dirKey.length() - 1);
- }
- return dirKey;
- }
-
- /**
- * Default method to creates a new OperationContext for the Azure Storage
- * operation that has listeners hooked to it that will update the metrics for
- * this file system. This method does not bind to receive send request
- * callbacks by default.
- *
- * @return The OperationContext object to use.
- */
- private OperationContext getInstrumentedContext() {
- // Default is to not bind to receive send callback events.
- return getInstrumentedContext(false);
- }
-
- /**
- * Creates a new OperationContext for the Azure Storage operation that has
- * listeners hooked to it that will update the metrics for this file system.
- *
- * @param bindConcurrentOOBIo
- * - bind to intercept send request call backs to handle OOB I/O.
- *
- * @return The OperationContext object to use.
- */
- private OperationContext getInstrumentedContext(boolean bindConcurrentOOBIo) {
-
- OperationContext operationContext = new OperationContext();
-
- // Set User-Agent
- operationContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() {
- @Override
- public void eventOccurred(SendingRequestEvent eventArg) {
- HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject();
- String userAgentInfo = String.format(Utility.LOCALE_US, "WASB/%s (%s) %s",
- VersionInfo.getVersion(), userAgentId, BaseRequest.getUserAgent());
- connection.setRequestProperty(Constants.HeaderConstants.USER_AGENT, userAgentInfo);
- }
- });
-
- if (selfThrottlingEnabled) {
- SelfThrottlingIntercept.hook(operationContext, selfThrottlingReadFactor,
- selfThrottlingWriteFactor);
- } else if (autoThrottlingEnabled) {
- ClientThrottlingIntercept.hook(operationContext);
- }
-
- if (bandwidthGaugeUpdater != null) {
- //bandwidthGaugeUpdater is null when we config to skip azure metrics
- ResponseReceivedMetricUpdater.hook(
- operationContext,
- instrumentation,
- bandwidthGaugeUpdater);
- }
-
- // Bind operation context to receive send request callbacks on this operation.
- // If reads concurrent to OOB writes are allowed, the interception will reset
- // the conditional header on all Azure blob storage read requests.
- if (bindConcurrentOOBIo) {
- SendRequestIntercept.bind(operationContext);
- }
-
- if (testHookOperationContext != null) {
- operationContext =
- testHookOperationContext.modifyOperationContext(operationContext);
- }
-
- ErrorMetricUpdater.hook(operationContext, instrumentation);
-
- // Return the operation context.
- return operationContext;
- }
-
- @Override
- public FileMetadata retrieveMetadata(String key) throws IOException {
-
- // Attempts to check status may occur before opening any streams so first,
- // check if a session exists, if not create a session with the Azure storage
- // server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- LOG.debug("Retrieving metadata for {}", key);
-
- try {
- if (checkContainer(ContainerAccessType.PureRead) == ContainerState.DoesntExist) {
- // The container doesn't exist, so spare some service calls and just
- // return null now.
- return null;
- }
-
- // Handle the degenerate cases where the key does not exist or the
- // key is a container.
- if (key.equals("/")) {
- // The key refers to root directory of container.
- // Set the modification time for root to zero.
- return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(),
- BlobMaterialization.Implicit, hadoopBlockSize);
- }
-
- CloudBlobWrapper blob = getBlobReference(key);
-
- // Download attributes and return file metadata only if the blob
- // exists.
- if (null != blob && blob.exists(getInstrumentedContext())) {
-
- LOG.debug("Found {} as an explicit blob. Checking if it's a file or folder.", key);
-
- try {
- // The blob exists, so capture the metadata from the blob
- // properties.
- blob.downloadAttributes(getInstrumentedContext());
- BlobProperties properties = blob.getProperties();
-
- if (retrieveFolderAttribute(blob)) {
- LOG.debug("{} is a folder blob.", key);
- return new FileMetadata(key, properties.getLastModified().getTime(),
- getPermissionStatus(blob), BlobMaterialization.Explicit, hadoopBlockSize);
- } else {
-
- LOG.debug("{} is a normal blob.", key);
-
- return new FileMetadata(
- key, // Always return denormalized key with metadata.
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob), hadoopBlockSize);
- }
- } catch(StorageException e){
- if (!NativeAzureFileSystemHelper.isFileNotFoundException(e)) {
- throw e;
- }
- }
- }
-
- // There is no file with that key name, but maybe it is a folder.
- // Query the underlying folder/container to list the blobs stored
- // there under that key.
- //
- Iterable<ListBlobItem> objects =
- listRootBlobs(
- key,
- true,
- EnumSet.of(BlobListingDetails.METADATA),
- null,
- getInstrumentedContext());
-
- // Check if the directory/container has the blob items.
- for (ListBlobItem blobItem : objects) {
- if (blobItem instanceof CloudBlockBlobWrapper
- || blobItem instanceof CloudPageBlobWrapper) {
- LOG.debug("Found blob as a directory-using this file under it to infer its properties {}",
- blobItem.getUri());
-
- blob = (CloudBlobWrapper) blobItem;
- // The key specifies a directory. Create a FileMetadata object which
- // specifies as such.
- BlobProperties properties = blob.getProperties();
-
- return new FileMetadata(key, properties.getLastModified().getTime(),
- getPermissionStatus(blob), BlobMaterialization.Implicit, hadoopBlockSize);
- }
- }
-
- // Return to caller with a null metadata object.
- return null;
-
- } catch (Exception e) {
- // Re-throw the exception as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public byte[] retrieveAttribute(String key, String attribute) throws IOException {
- try {
- checkContainer(ContainerAccessType.PureRead);
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
-
- String value = getMetadataAttribute(blob.getMetadata(),
- ensureValidAttributeName(attribute));
- value = decodeMetadataAttribute(value);
- return value == null ? null : value.getBytes(METADATA_ENCODING);
- } catch (Exception e) {
- throw new AzureException(e);
- }
- }
-
- @Override
- public void storeAttribute(String key, String attribute, byte[] value) throws IOException {
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
-
- String encodedValue = encodeMetadataAttribute(new String(value, METADATA_ENCODING));
- storeMetadataAttribute(blob, ensureValidAttributeName(attribute), encodedValue);
- blob.uploadMetadata(getInstrumentedContext());
- } catch (Exception e) {
- throw new AzureException(e);
- }
- }
-
- @Override
- public InputStream retrieve(String key) throws AzureException, IOException {
- return retrieve(key, 0);
- }
-
- @Override
- public InputStream retrieve(String key, long startByteOffset)
- throws AzureException, IOException {
- return retrieve(key, startByteOffset, Optional.empty());
- }
-
- @Override
- public InputStream retrieve(String key, long startByteOffset,
- Optional<Configuration> options) throws AzureException, IOException {
- try {
- // Check if a session exists, if not create a session with the
- // Azure storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
- checkContainer(ContainerAccessType.PureRead);
-
- InputStream inputStream = openInputStream(getBlobReference(key), options);
- if (startByteOffset > 0) {
- // Skip bytes and ignore return value. This is okay
- // because if you try to skip too far you will be positioned
- // at the end and reads will not return data.
- inputStream.skip(startByteOffset);
- }
- return inputStream;
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public FileMetadata[] list(String prefix, final int maxListingCount,
- final int maxListingDepth) throws IOException {
- return listInternal(prefix, maxListingCount, maxListingDepth);
- }
-
- private FileMetadata[] listInternal(String prefix, final int maxListingCount,
- final int maxListingDepth)
- throws IOException {
- try {
- checkContainer(ContainerAccessType.PureRead);
-
- if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
- prefix += PATH_DELIMITER;
- }
-
- // Enable flat listing option only if depth is unbounded and config
- // KEY_ENABLE_FLAT_LISTING is enabled.
- boolean enableFlatListing = false;
- if (maxListingDepth < 0 && sessionConfiguration.getBoolean(
- KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
- enableFlatListing = true;
- }
-
- Iterable<ListBlobItem> objects;
- if (prefix.equals("/")) {
- objects = listRootBlobs(true, enableFlatListing);
- } else {
- objects = listRootBlobs(prefix, true, enableFlatListing);
- }
-
- HashMap<String, FileMetadata> fileMetadata = new HashMap<>(256);
-
- for (ListBlobItem blobItem : objects) {
- // Check that the maximum listing count is not exhausted.
- //
- if (0 < maxListingCount
- && fileMetadata.size() >= maxListingCount) {
- break;
- }
-
- if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
- String blobKey = null;
- CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
- BlobProperties properties = blob.getProperties();
-
- // Determine format of the blob name depending on whether an absolute
- // path is being used or not.
- blobKey = normalizeKey(blob);
-
- FileMetadata metadata;
- if (retrieveFolderAttribute(blob)) {
- metadata = new FileMetadata(blobKey,
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- BlobMaterialization.Explicit,
- hadoopBlockSize);
- } else {
- metadata = new FileMetadata(
- blobKey,
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- hadoopBlockSize);
- }
- // Add the metadata but remove duplicates. Note that the azure
- // storage java SDK returns two types of entries: CloudBlobWrappter
- // and CloudDirectoryWrapper. In the case where WASB generated the
- // data, there will be an empty blob for each "directory", and we will
- // receive a CloudBlobWrapper. If there are also files within this
- // "directory", we will also receive a CloudDirectoryWrapper. To
- // complicate matters, the data may not be generated by WASB, in
- // which case we may not have an empty blob for each "directory".
- // So, sometimes we receive both a CloudBlobWrapper and a
- // CloudDirectoryWrapper for each directory, and sometimes we receive
- // one or the other but not both. We remove duplicates, but
- // prefer CloudBlobWrapper over CloudDirectoryWrapper.
- // Furthermore, it is very unfortunate that the list results are not
- // ordered, and it is a partial list which uses continuation. So
- // the HashMap is the best structure to remove the duplicates, despite
- // its potential large size.
- fileMetadata.put(blobKey, metadata);
-
- } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
- CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
- // Determine format of directory name depending on whether an absolute
- // path is being used or not.
- //
- String dirKey = normalizeKey(directory);
- // Strip the last /
- if (dirKey.endsWith(PATH_DELIMITER)) {
- dirKey = dirKey.substring(0, dirKey.length() - 1);
- }
-
- // Reached the targeted listing depth. Return metadata for the
- // directory using default permissions.
- //
- // Note: Something smarter should be done about permissions. Maybe
- // inherit the permissions of the first non-directory blob.
- // Also, getting a proper value for last-modified is tricky.
- FileMetadata directoryMetadata = new FileMetadata(dirKey, 0,
- defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit,
- hadoopBlockSize);
-
- // Add the directory metadata to the list only if it's not already
- // there. See earlier note, we prefer CloudBlobWrapper over
- // CloudDirectoryWrapper because it may have additional metadata (
- // properties and ACLs).
- if (!fileMetadata.containsKey(dirKey)) {
- fileMetadata.put(dirKey, directoryMetadata);
- }
-
- if (!enableFlatListing) {
- // Currently at a depth of one, decrement the listing depth for
- // sub-directories.
- buildUpList(directory, fileMetadata, maxListingCount,
- maxListingDepth - 1);
- }
- }
- }
- return fileMetadata.values().toArray(new FileMetadata[fileMetadata.size()]);
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
- //
- throw new AzureException(e);
- }
- }
-
- /**
- * Build up a metadata list of blobs in an Azure blob directory. This method
- * uses a in-order first traversal of blob directory structures to maintain
- * the sorted order of the blob names.
- *
- * @param aCloudBlobDirectory Azure blob directory
- * @param metadataHashMap a map of file metadata objects for each
- * non-directory blob.
- * @param maxListingCount maximum length of the built up list.
- */
- private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory,
- HashMap<String, FileMetadata> metadataHashMap, final int maxListingCount,
- final int maxListingDepth) throws Exception {
-
- // Push the blob directory onto the stack.
- //
- AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack =
- new AzureLinkedStack<Iterator<ListBlobItem>>();
-
- Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null,
- false, EnumSet.of(BlobListingDetails.METADATA), null,
- getInstrumentedContext());
- Iterator<ListBlobItem> blobItemIterator = blobItems.iterator();
-
- if (0 == maxListingDepth || 0 == maxListingCount) {
- // Recurrence depth and listing count are already exhausted. Return
- // immediately.
- return;
- }
-
- // The directory listing depth is unbounded if the maximum listing depth
- // is negative.
- final boolean isUnboundedDepth = (maxListingDepth < 0);
-
- // Reset the current directory listing depth.
- int listingDepth = 1;
-
- // Loop until all directories have been traversed in-order. Loop only
- // the following conditions are satisfied:
- // (1) The stack is not empty, and
- // (2) maxListingCount > 0 implies that the number of items in the
- // metadata list is less than the max listing count.
- while (null != blobItemIterator
- && (maxListingCount <= 0 || metadataHashMap.size() < maxListingCount)) {
- while (blobItemIterator.hasNext()) {
- // Check if the count of items on the list exhausts the maximum
- // listing count.
- //
- if (0 < maxListingCount && metadataHashMap.size() >= maxListingCount) {
- break;
- }
-
- ListBlobItem blobItem = blobItemIterator.next();
-
- // Add the file metadata to the list if this is not a blob
- // directory item.
- //
- if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
- String blobKey = null;
- CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
- BlobProperties properties = blob.getProperties();
-
- // Determine format of the blob name depending on whether an absolute
- // path is being used or not.
- blobKey = normalizeKey(blob);
-
- FileMetadata metadata;
- if (retrieveFolderAttribute(blob)) {
- metadata = new FileMetadata(blobKey,
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- BlobMaterialization.Explicit,
- hadoopBlockSize);
- } else {
- metadata = new FileMetadata(
- blobKey,
- getDataLength(blob, properties),
- properties.getLastModified().getTime(),
- getPermissionStatus(blob),
- hadoopBlockSize);
- }
-
- // Add the metadata but remove duplicates. Note that the azure
- // storage java SDK returns two types of entries: CloudBlobWrappter
- // and CloudDirectoryWrapper. In the case where WASB generated the
- // data, there will be an empty blob for each "directory", and we will
- // receive a CloudBlobWrapper. If there are also files within this
- // "directory", we will also receive a CloudDirectoryWrapper. To
- // complicate matters, the data may not be generated by WASB, in
- // which case we may not have an empty blob for each "directory".
- // So, sometimes we receive both a CloudBlobWrapper and a
- // CloudDirectoryWrapper for each directory, and sometimes we receive
- // one or the other but not both. We remove duplicates, but
- // prefer CloudBlobWrapper over CloudDirectoryWrapper.
- // Furthermore, it is very unfortunate that the list results are not
- // ordered, and it is a partial list which uses continuation. So
- // the HashMap is the best structure to remove the duplicates, despite
- // its potential large size.
- metadataHashMap.put(blobKey, metadata);
- } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
- CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
-
- // This is a directory blob, push the current iterator onto
- // the stack of iterators and start iterating through the current
- // directory.
- if (isUnboundedDepth || maxListingDepth > listingDepth) {
- // Push the current directory on the stack and increment the listing
- // depth.
- dirIteratorStack.push(blobItemIterator);
- ++listingDepth;
-
- // The current blob item represents the new directory. Get
- // an iterator for this directory and continue by iterating through
- // this directory.
- blobItems = directory.listBlobs(null, false,
- EnumSet.noneOf(BlobListingDetails.class), null,
- getInstrumentedContext());
- blobItemIterator = blobItems.iterator();
- } else {
- // Determine format of directory name depending on whether an
- // absolute path is being used or not.
- String dirKey = normalizeKey(directory);
-
- // Add the directory metadata to the list only if it's not already
- // there. See earlier note, we prefer CloudBlobWrapper over
- // CloudDirectoryWrapper because it may have additional metadata (
- // properties and ACLs).
- if (!metadataHashMap.containsKey(dirKey)) {
-
- // Reached the targeted listing depth. Return metadata for the
- // directory using default permissions.
- //
- // Note: Something smarter should be done about permissions. Maybe
- // inherit the permissions of the first non-directory blob.
- // Also, getting a proper value for last-modified is tricky.
- //
- FileMetadata directoryMetadata = new FileMetadata(dirKey,
- 0,
- defaultPermissionNoBlobMetadata(),
- BlobMaterialization.Implicit,
- hadoopBlockSize);
-
- // Add the directory metadata to the list.
- metadataHashMap.put(dirKey, directoryMetadata);
- }
- }
- }
- }
-
- // Traversal of directory tree
-
- // Check if the iterator stack is empty. If it is set the next blob
- // iterator to null. This will act as a terminator for the for-loop.
- // Otherwise pop the next iterator from the stack and continue looping.
- //
- if (dirIteratorStack.isEmpty()) {
- blobItemIterator = null;
- } else {
- // Pop the next directory item from the stack and decrement the
- // depth.
- blobItemIterator = dirIteratorStack.pop();
- --listingDepth;
-
- // Assertion: Listing depth should not be less than zero.
- if (listingDepth < 0) {
- throw new AssertionError("Non-negative listing depth expected");
- }
- }
- }
- }
-
- /**
- * Return the actual data length of the blob with the specified properties.
- * If it is a page blob, you can't rely on the length from the properties
- * argument and you must get it from the file. Otherwise, you can.
- */
- private long getDataLength(CloudBlobWrapper blob, BlobProperties properties)
- throws AzureException {
- if (blob instanceof CloudPageBlobWrapper) {
- try {
- return PageBlobInputStream.getPageBlobDataSize((CloudPageBlobWrapper) blob,
- getInstrumentedContext(
- isConcurrentOOBAppendAllowed()));
- } catch (Exception e) {
- throw new AzureException(
- "Unexpected exception getting page blob actual data size.", e);
- }
- }
- return properties.getLength();
- }
-
- /**
- * Deletes the given blob, taking special care that if we get a
- * blob-not-found exception upon retrying the operation, we just
- * swallow the error since what most probably happened is that
- * the first operation succeeded on the server.
- * @param blob The blob to delete.
- * @param lease Azure blob lease, or null if no lease is to be used.
- * @throws StorageException
- */
- private void safeDelete(CloudBlobWrapper blob, SelfRenewingLease lease) throws StorageException {
- OperationContext operationContext = getInstrumentedContext();
- try {
- blob.delete(operationContext, lease);
- } catch (StorageException e) {
- if (!NativeAzureFileSystemHelper.isFileNotFoundException(e)) {
- LOG.error("Encountered Storage Exception for delete on Blob: {}"
- + ", Exception Details: {} Error Code: {}",
- blob.getUri(), e.getMessage(), e.getErrorCode());
- }
- // On exception, check that if:
- // 1. It's a BlobNotFound exception AND
- // 2. It got there after one-or-more retries THEN
- // we swallow the exception.
- if (e.getErrorCode() != null
- && "BlobNotFound".equals(e.getErrorCode())
- && operationContext.getRequestResults().size() > 1
- && operationContext.getRequestResults().get(0).getException() != null) {
- LOG.debug("Swallowing delete exception on retry: {}", e.getMessage());
- return;
- } else {
- throw e;
- }
- } finally {
- if (lease != null) {
- lease.free();
- }
- }
- }
-
- /**
- * API implementation to delete a blob in the back end azure storage.
- */
- @Override
- public boolean delete(String key, SelfRenewingLease lease) throws IOException {
- try {
- if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) {
- // Container doesn't exist, no need to do anything
- return true;
- }
- // Get the blob reference and delete it.
- CloudBlobWrapper blob = getBlobReference(key);
- safeDelete(blob, lease);
- return true;
- } catch (Exception e) {
- if (e instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException(
- (StorageException) e)) {
- // the file or directory does not exist
- return false;
- }
- throw new AzureException(e);
- }
- }
-
- /**
- * API implementation to delete a blob in the back end azure storage.
- */
- @Override
- public boolean delete(String key) throws IOException {
- try {
- return delete(key, null);
- } catch (IOException e) {
- Throwable t = e.getCause();
- if (t instanceof StorageException) {
- StorageException se = (StorageException) t;
- if ("LeaseIdMissing".equals(se.getErrorCode())){
- SelfRenewingLease lease = null;
- try {
- lease = acquireLease(key);
- return delete(key, lease);
- } catch (AzureException e3) {
- LOG.warn("Got unexpected exception trying to acquire lease on "
- + key + "." + e3.getMessage());
- throw e3;
- } finally {
- try {
- if (lease != null){
- lease.free();
- }
- } catch (Exception e4){
- LOG.error("Unable to free lease on " + key, e4);
- }
- }
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- }
- }
-
- @Override
- public void rename(String srcKey, String dstKey) throws IOException {
- rename(srcKey, dstKey, false, null, true);
- }
-
- @Override
- public void rename(String srcKey, String dstKey, boolean acquireLease,
- SelfRenewingLease existingLease) throws IOException {
- rename(srcKey, dstKey, acquireLease, existingLease, true);
- }
-
- @Override
- public void rename(String srcKey, String dstKey, boolean acquireLease,
- SelfRenewingLease existingLease, boolean overwriteDestination) throws IOException {
-
- LOG.debug("Moving {} to {}", srcKey, dstKey);
-
- if (acquireLease && existingLease != null) {
- throw new IOException("Cannot acquire new lease if one already exists.");
- }
-
- CloudBlobWrapper srcBlob = null;
- CloudBlobWrapper dstBlob = null;
- SelfRenewingLease lease = null;
- try {
- // Attempts rename may occur before opening any streams so first,
- // check if a session exists, if not create a session with the Azure
- // storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- checkContainer(ContainerAccessType.ReadThenWrite);
- // Get the source blob and assert its existence. If the source key
- // needs to be normalized then normalize it.
- //
-
- srcBlob = getBlobReference(srcKey);
- if (!srcBlob.exists(getInstrumentedContext())) {
- throw new AzureException("Source blob " + srcKey + " does not exist.");
- }
-
- /**
- * Conditionally get a lease on the source blob to prevent other writers
- * from changing it. This is used for correctness in HBase when log files
- * are renamed. It generally should do no harm other than take a little
- * more time for other rename scenarios. When the HBase master renames a
- * log file folder, the lease locks out other writers. This
- * prevents a region server that the master thinks is dead, but is still
- * alive, from committing additional updates. This is different than
- * when HBase runs on HDFS, where the region server recovers the lease
- * on a log file, to gain exclusive access to it, before it splits it.
- */
- if (acquireLease) {
- lease = srcBlob.acquireLease();
- } else if (existingLease != null) {
- lease = existingLease;
- }
-
- // Get the destination blob. The destination key always needs to be
- // normalized.
- //
- dstBlob = getBlobReference(dstKey);
-
- // Rename the source blob to the destination blob by copying it to
- // the destination blob then deleting it.
- //
- // Copy blob operation in Azure storage is very costly. It will be highly
- // likely throttled during Azure storage gc. Short term fix will be using
- // a more intensive exponential retry policy when the cluster is getting
- // throttled.
- try {
- dstBlob.startCopyFromBlob(srcBlob, null,
- getInstrumentedContext(), overwriteDestination);
- } catch (StorageException se) {
- if (se.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
- int copyBlobMinBackoff = sessionConfiguration.getInt(
- KEY_COPYBLOB_MIN_BACKOFF_INTERVAL,
- DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL);
-
- int copyBlobMaxBackoff = sessionConfiguration.getInt(
- KEY_COPYBLOB_MAX_BACKOFF_INTERVAL,
- DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL);
-
- int copyBlobDeltaBackoff = sessionConfiguration.getInt(
- KEY_COPYBLOB_BACKOFF_INTERVAL,
- DEFAULT_COPYBLOB_BACKOFF_INTERVAL);
-
- int copyBlobMaxRetries = sessionConfiguration.getInt(
- KEY_COPYBLOB_MAX_IO_RETRIES,
- DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS);
-
- BlobRequestOptions options = new BlobRequestOptions();
- options.setRetryPolicyFactory(new RetryExponentialRetry(
- copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff,
- copyBlobMaxRetries));
- dstBlob.startCopyFromBlob(srcBlob, options,
- getInstrumentedContext(), overwriteDestination);
- } else {
- throw se;
- }
- }
- waitForCopyToComplete(dstBlob, getInstrumentedContext());
- safeDelete(srcBlob, lease);
- } catch (StorageException e) {
- if (e.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
- LOG.warn("Rename: CopyBlob: StorageException: ServerBusy: Retry complete, will attempt client side copy for page blob");
- InputStream ipStream = null;
- OutputStream opStream = null;
- try {
- if (srcBlob.getProperties().getBlobType() == BlobType.PAGE_BLOB){
- ipStream = openInputStream(srcBlob, Optional.empty());
- opStream = openOutputStream(dstBlob);
- byte[] buffer = new byte[PageBlobFormatHelpers.PAGE_SIZE];
- int len;
- while ((len = ipStream.read(buffer)) != -1) {
- opStream.write(buffer, 0, len);
- }
- opStream.flush();
- opStream.close();
- ipStream.close();
- } else {
- throw new AzureException(e);
- }
- safeDelete(srcBlob, lease);
- } catch(StorageException se) {
- LOG.warn("Rename: CopyBlob: StorageException: Failed");
- throw new AzureException(se);
- } finally {
- IOUtils.closeStream(ipStream);
- IOUtils.closeStream(opStream);
- }
- } else {
- throw new AzureException(e);
- }
- } catch (URISyntaxException e) {
- // Re-throw exception as an Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- private void waitForCopyToComplete(CloudBlobWrapper blob, OperationContext opContext){
- boolean copyInProgress = true;
- while (copyInProgress) {
- try {
- blob.downloadAttributes(opContext);
- }
- catch (StorageException se){
- }
-
- // test for null because mocked filesystem doesn't know about copystates yet.
- copyInProgress = (blob.getCopyState() != null && blob.getCopyState().getStatus() == CopyStatus.PENDING);
- if (copyInProgress) {
- try {
- Thread.sleep(1000);
- }
- catch (InterruptedException ie){
- //ignore
- }
- }
- }
- }
-
- /**
- * Checks whether an explicit file/folder exists.
- * This is used by redo of atomic rename.
- * There was a bug(apache jira HADOOP-12780) during atomic rename if
- * process crashes after an inner directory has been renamed but still
- * there are file under that directory to be renamed then after the
- * process comes again it tries to redo the renames. It checks whether
- * the directory exists or not by calling filesystem.exist.
- * But filesystem.Exists will treat that directory as implicit directory
- * and return true as file exists under that directory. So It will try
- * try to rename that directory and will fail as the corresponding blob
- * does not exist. So this method explicitly checks for the blob.
- */
- @Override
- public boolean explicitFileExists(String key) throws AzureException {
- CloudBlobWrapper blob;
- try {
- blob = getBlobReference(key);
- if (null != blob && blob.exists(getInstrumentedContext())) {
- return true;
- }
-
- return false;
- } catch (StorageException e) {
- throw new AzureException(e);
- } catch (URISyntaxException e) {
- throw new AzureException(e);
- }
- }
-
- /**
- * Changes the permission status on the given key.
- */
- @Override
- public void changePermissionStatus(String key, PermissionStatus newPermission)
- throws AzureException {
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- blob.downloadAttributes(getInstrumentedContext());
- storePermissionStatus(blob, newPermission);
- blob.uploadMetadata(getInstrumentedContext());
- } catch (Exception e) {
- throw new AzureException(e);
- }
- }
-
- @Override
- public void purge(String prefix) throws IOException {
- try {
-
- // Attempts to purge may occur before opening any streams so first,
- // check if a session exists, if not create a session with the Azure
- // storage server.
- if (null == storageInteractionLayer) {
- final String errMsg = String.format(
- "Storage session expected for URI '%s' but does not exist.",
- sessionUri);
- throw new AssertionError(errMsg);
- }
-
- if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) {
- // Container doesn't exist, no need to do anything.
- return;
- }
- // Get all blob items with the given prefix from the container and delete
- // them.
- Iterable<ListBlobItem> objects = listRootBlobs(prefix, false, false);
- for (ListBlobItem blobItem : objects) {
- ((CloudBlob) blobItem).delete(DeleteSnapshotsOption.NONE, null, null,
- getInstrumentedContext());
- }
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
- //
- throw new AzureException(e);
- }
- }
-
- /**
- * Get a lease on the blob identified by key. This lease will be renewed
- * indefinitely by a background thread.
- */
- @Override
- public SelfRenewingLease acquireLease(String key) throws AzureException {
- LOG.debug("acquiring lease on {}", key);
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- return blob.acquireLease();
- }
- catch (Exception e) {
-
- // Caught exception while attempting to get lease. Re-throw as an
- // Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public void updateFolderLastModifiedTime(String key, Date lastModified,
- SelfRenewingLease folderLease)
- throws AzureException {
- try {
- checkContainer(ContainerAccessType.ReadThenWrite);
- CloudBlobWrapper blob = getBlobReference(key);
- //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified
- //timestamp to current time
- blob.uploadProperties(getInstrumentedContext(), folderLease);
- } catch (Exception e) {
-
- // Caught exception while attempting to update the properties. Re-throw as an
- // Azure storage exception.
- throw new AzureException(e);
- }
- }
-
- @Override
- public void updateFolderLastModifiedTime(String key,
- SelfRenewingLease folderLease) throws AzureException {
- final Calendar lastModifiedCalendar = Calendar
- .getInstance(Utility.LOCALE_US);
- lastModifiedCalendar.setTimeZone(Utility.UTC_ZONE);
- Date lastModified = lastModifiedCalendar.getTime();
- updateFolderLastModifiedTime(key, lastModified, folderLease);
- }
-
- @Override
- public void dump() throws IOException {
- }
-
- @Override
- public void close() {
- if (bandwidthGaugeUpdater != null) {
- bandwidthGaugeUpdater.close();
- bandwidthGaugeUpdater = null;
- }
- }
-
- // Finalizer to ensure complete shutdown
- @Override
- protected void finalize() throws Throwable {
- LOG.debug("finalize() called");
- close();
- super.finalize();
- }
-
- @Override
- public DataOutputStream retrieveAppendStream(String key, int bufferSize) throws IOException {
-
- try {
-
- if (isPageBlobKey(key)) {
- throw new UnsupportedOperationException("Append not supported for Page Blobs");
- }
-
- CloudBlobWrapper blob = this.container.getBlockBlobReference(key);
-
- OutputStream outputStream;
-
- BlockBlobAppendStream blockBlobOutputStream = new BlockBlobAppendStream(
- (CloudBlockBlobWrapper) blob,
- key,
- bufferSize,
- isBlockBlobWithCompactionKey(key),
- getInstrumentedContext());
-
- outputStream = blockBlobOutputStream;
-
- DataOutputStream dataOutStream = new SyncableDataOutputStream(
- outputStream);
-
- return dataOutStream;
- } catch(Exception ex) {
- throw new AzureException(ex);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java
deleted file mode 100644
index a1f82428b49..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobMaterialization.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Indicates whether there are actual blobs indicating the existence of
- * directories or whether we're inferring their existence from them having files
- * in there.
- */
-@InterfaceAudience.Private
-enum BlobMaterialization {
- /**
- * Indicates a directory that isn't backed by an actual blob, but its
- * existence is implied by the fact that there are files in there. For
- * example, if the blob /a/b exists then it implies the existence of the /a
- * directory if there's no /a blob indicating it.
- */
- Implicit,
- /**
- * Indicates that the directory is backed by an actual blob that has the
- * isFolder metadata on it.
- */
- Explicit,
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java
deleted file mode 100644
index 6da64e124ef..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlobOperationDescriptor.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import com.microsoft.azure.storage.Constants.HeaderConstants;
-import org.apache.hadoop.classification.InterfaceAudience;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Determines the operation type (PutBlock, PutPage, GetBlob, etc) of Azure
- * Storage operations. This is used by the handlers of the SendingRequestEvent
- * and ResponseReceivedEvent exposed by the Azure Storage SDK to identify
- * operation types (since the type of operation is not exposed by the SDK).
- */
-@InterfaceAudience.Private
-final class BlobOperationDescriptor {
-
- private BlobOperationDescriptor() {
- // hide default constructor
- }
-
- /**
- * Gets the content length for the Azure Storage operation from the
- * 'x-ms-range' header, if set.
- * @param range the value of the 'x-ms-range' header.
- * @return the content length, or zero if not set.
- */
- private static long getContentLengthIfKnown(String range) {
- long contentLength = 0;
- // Format is "bytes=%d-%d"
- if (range != null && range.startsWith("bytes=")) {
- String[] offsets = range.substring("bytes=".length()).split("-");
- if (offsets.length == 2) {
- contentLength = Long.parseLong(offsets[1]) - Long.parseLong(offsets[0])
- + 1;
- }
- }
- return contentLength;
- }
-
- /**
- * Gets the content length for the Azure Storage operation, or returns zero if
- * unknown.
- * @param conn the connection object for the Azure Storage operation.
- * @param operationType the Azure Storage operation type.
- * @return the content length, or zero if unknown.
- */
- static long getContentLengthIfKnown(HttpURLConnection conn,
- OperationType operationType) {
- long contentLength = 0;
- switch (operationType) {
- case AppendBlock:
- case PutBlock:
- String lengthString = conn.getRequestProperty(
- HeaderConstants.CONTENT_LENGTH);
- contentLength = (lengthString != null)
- ? Long.parseLong(lengthString)
- : 0;
- break;
- case PutPage:
- case GetBlob:
- contentLength = BlobOperationDescriptor.getContentLengthIfKnown(
- conn.getRequestProperty("x-ms-range"));
- break;
- default:
- break;
- }
- return contentLength;
- }
-
- /**
- * Gets the operation type of an Azure Storage operation.
- *
- * @param conn the connection object for the Azure Storage operation.
- * @return the operation type.
- */
- static OperationType getOperationType(HttpURLConnection conn) {
- OperationType operationType = OperationType.Unknown;
- String method = conn.getRequestMethod();
- String compValue = getQueryParameter(conn.getURL(),
- "comp");
-
- if (method.equalsIgnoreCase("PUT")) {
- if (compValue != null) {
- switch (compValue) {
- case "metadata":
- operationType = OperationType.SetMetadata;
- break;
- case "properties":
- operationType = OperationType.SetProperties;
- break;
- case "block":
- operationType = OperationType.PutBlock;
- break;
- case "page":
- String pageWrite = conn.getRequestProperty("x-ms-page-write");
- if (pageWrite != null && pageWrite.equalsIgnoreCase(
- "UPDATE")) {
- operationType = OperationType.PutPage;
- }
- break;
- case "appendblock":
- operationType = OperationType.AppendBlock;
- break;
- case "blocklist":
- operationType = OperationType.PutBlockList;
- break;
- default:
- break;
- }
- } else {
- String blobType = conn.getRequestProperty("x-ms-blob-type");
- if (blobType != null
- && (blobType.equalsIgnoreCase("PageBlob")
- || blobType.equalsIgnoreCase("BlockBlob")
- || blobType.equalsIgnoreCase("AppendBlob"))) {
- operationType = OperationType.CreateBlob;
- } else if (blobType == null) {
- String resType = getQueryParameter(conn.getURL(),
- "restype");
- if (resType != null
- && resType.equalsIgnoreCase("container")) {
- operationType = operationType.CreateContainer;
- }
- }
- }
- } else if (method.equalsIgnoreCase("GET")) {
- if (compValue != null) {
- switch (compValue) {
- case "list":
- operationType = OperationType.ListBlobs;
- break;
-
- case "metadata":
- operationType = OperationType.GetMetadata;
- break;
- case "blocklist":
- operationType = OperationType.GetBlockList;
- break;
- case "pagelist":
- operationType = OperationType.GetPageList;
- break;
- default:
- break;
- }
- } else if (conn.getRequestProperty("x-ms-range") != null) {
- operationType = OperationType.GetBlob;
- }
- } else if (method.equalsIgnoreCase("HEAD")) {
- operationType = OperationType.GetProperties;
- } else if (method.equalsIgnoreCase("DELETE")) {
- String resType = getQueryParameter(conn.getURL(),
- "restype");
- if (resType != null
- && resType.equalsIgnoreCase("container")) {
- operationType = operationType.DeleteContainer;
- } else {
- operationType = OperationType.DeleteBlob;
- }
- }
- return operationType;
- }
-
- private static String getQueryParameter(URL url, String queryParameterName) {
- String query = (url != null) ? url.getQuery(): null;
-
- if (query == null) {
- return null;
- }
-
- String searchValue = queryParameterName + "=";
-
- int offset = query.indexOf(searchValue);
- String value = null;
- if (offset != -1) {
- int beginIndex = offset + searchValue.length();
- int endIndex = query.indexOf('&', beginIndex);
- value = (endIndex == -1)
- ? query.substring(beginIndex)
- : query.substring(beginIndex, endIndex);
- }
- return value;
- }
-
- @InterfaceAudience.Private
- enum OperationType {
- AppendBlock,
- CreateBlob,
- CreateContainer,
- DeleteBlob,
- DeleteContainer,
- GetBlob,
- GetBlockList,
- GetMetadata,
- GetPageList,
- GetProperties,
- ListBlobs,
- PutBlock,
- PutBlockList,
- PutPage,
- SetMetadata,
- SetProperties,
- Unknown
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
deleted file mode 100644
index c19bbe48fc3..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ /dev/null
@@ -1,1141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-import java.util.Random;
-import java.util.concurrent.ConcurrentLinkedDeque;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.util.Preconditions;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-import org.apache.hadoop.io.ElasticByteBufferPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.StorageErrorCodeStrings;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.BlockSearchMode;
-
-/**
- * Stream object that implements append for Block Blobs in WASB.
- *
- * The stream object implements hflush/hsync and block compaction. Block
- * compaction is the process of replacing a sequence of small blocks with one
- * big block. Azure Block blobs supports up to 50000 blocks and every
- * hflush/hsync generates one block. When the number of blocks is above 32000,
- * the process of compaction decreases the total number of blocks, if possible.
- * If compaction is disabled, hflush/hsync are empty functions.
- *
- * The stream object uses background threads for uploading the blocks and the
- * block blob list. Blocks can be uploaded concurrently. However, when the block
- * list is uploaded, block uploading should stop. If a block is uploaded before
- * the block list and the block id is not in the list, the block will be lost.
- * If the block is uploaded after the block list and the block id is in the
- * list, the block list upload will fail. The exclusive access for the block
- * list upload is managed by uploadingSemaphore.
- */
-
-public class BlockBlobAppendStream extends OutputStream implements Syncable,
- StreamCapabilities {
-
- /**
- * The name of the blob/file.
- */
- private final String key;
-
- /**
- * This variable tracks if this is new blob or existing one.
- */
- private boolean blobExist;
-
- /**
- * When the blob exist, to to prevent concurrent write we take a lease.
- * Taking a lease is not necessary for new blobs.
- */
- private SelfRenewingLease lease = null;
-
- /**
- * The support for process of compaction is optional.
- */
- private final boolean compactionEnabled;
-
- /**
- * The number of blocks above each block compaction is triggered.
- */
- private static final int DEFAULT_ACTIVATE_COMPACTION_BLOCK_COUNT = 32000;
-
- /**
- * The number of blocks above each block compaction is triggered.
- */
- private int activateCompactionBlockCount
- = DEFAULT_ACTIVATE_COMPACTION_BLOCK_COUNT;
-
- /**
- * The size of the output buffer. Writes store the data in outBuffer until
- * either the size is above maxBlockSize or hflush/hsync is called.
- */
- private final AtomicInteger maxBlockSize;
-
- /**
- * The current buffer where writes are stored.
- */
- private ByteBuffer outBuffer;
-
- /**
- * The size of the blob that has been successfully stored in the Azure Blob
- * service.
- */
- private final AtomicLong committedBlobLength = new AtomicLong(0);
-
- /**
- * Position of last block in the blob.
- */
- private volatile long blobLength = 0;
-
- /**
- * Minutes waiting before the close operation timed out.
- */
- private static final int CLOSE_UPLOAD_DELAY = 10;
-
- /**
- * Keep alive time for the threadpool.
- */
- private static final int THREADPOOL_KEEP_ALIVE = 30;
- /**
- * Azure Block Blob used for the stream.
- */
- private final CloudBlockBlobWrapper blob;
-
- /**
- * Azure Storage operation context.
- */
- private final OperationContext opContext;
-
- /**
- * Commands send from client calls to the background thread pool.
- */
- private abstract class UploadCommand {
-
- // the blob offset for the command
- private final long commandBlobOffset;
-
- // command completion latch
- private final CountDownLatch completed = new CountDownLatch(1);
-
- UploadCommand(long offset) {
- this.commandBlobOffset = offset;
- }
-
- long getCommandBlobOffset() {
- return commandBlobOffset;
- }
-
- void await() throws InterruptedException {
- completed.await();
- }
-
- void awaitAsDependent() throws InterruptedException {
- await();
- }
-
- void setCompleted() {
- completed.countDown();
- }
-
- void execute() throws InterruptedException, IOException {}
-
- void dump() {}
- }
-
- /**
- * The list of recent commands. Before block list is committed, all the block
- * listed in the list must be uploaded. activeBlockCommands is used for
- * enumerating the blocks and waiting on the latch until the block is
- * uploaded.
- */
- private final ConcurrentLinkedQueue<UploadCommand> activeBlockCommands
- = new ConcurrentLinkedQueue<>();
-
- /**
- * Variable to track if the stream has been closed.
- */
- private volatile boolean closed = false;
-
- /**
- * First IOException encountered.
- */
- private final AtomicReference<IOException> firstError
- = new AtomicReference<>();
-
- /**
- * Flag set when the first error has been thrown.
- */
- private boolean firstErrorThrown = false;
-
- /**
- * Semaphore for serializing block uploads with NativeAzureFileSystem.
- *
- * The semaphore starts with number of permits equal to the number of block
- * upload threads. Each block upload thread needs one permit to start the
- * upload. The put block list acquires all the permits before the block list
- * is committed.
- */
- private final Semaphore uploadingSemaphore = new Semaphore(
- MAX_NUMBER_THREADS_IN_THREAD_POOL,
- true);
-
- /**
- * Queue storing buffers with the size of the Azure block ready for
- * reuse. The pool allows reusing the blocks instead of allocating new
- * blocks. After the data is sent to the service, the buffer is returned
- * back to the queue
- */
- private final ElasticByteBufferPool poolReadyByteBuffers
- = new ElasticByteBufferPool();
-
- /**
- * The blob's block list.
- */
- private final List<BlockEntry> blockEntries = new ArrayList<>(
- DEFAULT_CAPACITY_BLOCK_ENTRIES);
-
- private static final int DEFAULT_CAPACITY_BLOCK_ENTRIES = 1024;
-
- /**
- * The uncommitted blob's block list.
- */
- private final ConcurrentLinkedDeque<BlockEntry> uncommittedBlockEntries
- = new ConcurrentLinkedDeque<>();
-
- /**
- * Variable to hold the next block id to be used for azure storage blocks.
- */
- private static final int UNSET_BLOCKS_COUNT = -1;
- private long nextBlockCount = UNSET_BLOCKS_COUNT;
-
- /**
- * Variable to hold the block id prefix to be used for azure storage blocks.
- */
- private String blockIdPrefix = null;
-
- /**
- * Maximum number of threads in block upload thread pool.
- */
- private static final int MAX_NUMBER_THREADS_IN_THREAD_POOL = 4;
-
- /**
- * Number of times block upload needs is retried.
- */
- private static final int MAX_BLOCK_UPLOAD_RETRIES = 3;
-
- /**
- * Wait time between block upload retries in milliseconds.
- */
- private static final int BLOCK_UPLOAD_RETRY_INTERVAL = 1000;
-
- /**
- * Logger.
- */
- private static final Logger LOG =
- LoggerFactory.getLogger(BlockBlobAppendStream.class);
-
- /**
- * The absolute maximum of blocks for a blob. It includes committed and
- * temporary blocks.
- */
- private static final int MAX_BLOCK_COUNT = 100000;
-
- /**
- * The upload thread pool executor.
- */
- private ThreadPoolExecutor ioThreadPool;
-
- /**
- * Azure Storage access conditions for the blob.
- */
- private final AccessCondition accessCondition = new AccessCondition();
-
- /**
- * Atomic integer to provide thread id for thread names for uploader threads.
- */
- private final AtomicInteger threadSequenceNumber;
-
- /**
- * Prefix to be used for thread names for uploader threads.
- */
- private static final String THREAD_ID_PREFIX = "append-blockblob";
-
- /**
- * BlockBlobAppendStream constructor.
- *
- * @param blob
- * Azure Block Blob
- * @param aKey
- * blob's name
- * @param bufferSize
- * the maximum size of a blob block.
- * @param compactionEnabled
- * is the compaction process enabled for this blob
- * @param opContext
- * Azure Store operation context for the blob
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream cannot be used for append operations
- */
- public BlockBlobAppendStream(final CloudBlockBlobWrapper blob,
- final String aKey,
- final int bufferSize,
- final boolean compactionEnabled,
- final OperationContext opContext)
- throws IOException {
-
- Preconditions.checkArgument(StringUtils.isNotEmpty(aKey));
- Preconditions.checkArgument(bufferSize >= 0);
-
- this.blob = blob;
- this.opContext = opContext;
- this.key = aKey;
- this.maxBlockSize = new AtomicInteger(bufferSize);
- this.threadSequenceNumber = new AtomicInteger(0);
- this.blockIdPrefix = null;
- this.compactionEnabled = compactionEnabled;
- this.blobExist = true;
- this.outBuffer = poolReadyByteBuffers.getBuffer(false, maxBlockSize.get());
-
- try {
- // download the block list
- blockEntries.addAll(
- blob.downloadBlockList(
- BlockListingFilter.COMMITTED,
- new BlobRequestOptions(),
- opContext));
-
- blobLength = blob.getProperties().getLength();
-
- committedBlobLength.set(blobLength);
-
- // Acquiring lease on the blob.
- lease = new SelfRenewingLease(blob, true);
- accessCondition.setLeaseID(lease.getLeaseID());
-
- } catch (StorageException ex) {
- if (ex.getErrorCode().equals(StorageErrorCodeStrings.BLOB_NOT_FOUND)) {
- blobExist = false;
- }
- else if (ex.getErrorCode().equals(
- StorageErrorCodeStrings.LEASE_ALREADY_PRESENT)) {
- throw new AzureException(
- "Unable to set Append lease on the Blob: " + ex, ex);
- }
- else {
- LOG.debug(
- "Encountered storage exception."
- + " StorageException : {} ErrorCode : {}",
- ex,
- ex.getErrorCode());
- throw new AzureException(ex);
- }
- }
-
- setBlocksCountAndBlockIdPrefix(blockEntries);
-
- this.ioThreadPool = new ThreadPoolExecutor(
- MAX_NUMBER_THREADS_IN_THREAD_POOL,
- MAX_NUMBER_THREADS_IN_THREAD_POOL,
- THREADPOOL_KEEP_ALIVE,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(),
- new UploaderThreadFactory());
- }
-
- /**
- * Set payload size of the stream.
- * It is intended to be used for unit testing purposes only.
- */
- @VisibleForTesting
- synchronized void setMaxBlockSize(int size) {
- maxBlockSize.set(size);
-
- // it is for testing only so we can abandon the previously allocated
- // payload
- this.outBuffer = ByteBuffer.allocate(maxBlockSize.get());
- }
-
- /**
- * Set compaction parameters.
- * It is intended to be used for unit testing purposes only.
- */
- @VisibleForTesting
- void setCompactionBlockCount(int activationCount) {
- activateCompactionBlockCount = activationCount;
- }
-
- /**
- * Get the list of block entries. It is used for testing purposes only.
- * @return List of block entries.
- */
- @VisibleForTesting
- List<BlockEntry> getBlockList() throws StorageException, IOException {
- return blob.downloadBlockList(
- BlockListingFilter.COMMITTED,
- new BlobRequestOptions(),
- opContext);
- }
-
- /**
- * Writes the specified byte to this output stream. The general contract for
- * write is that one byte is written to the output stream. The byte to be
- * written is the eight low-order bits of the argument b. The 24 high-order
- * bits of b are ignored.
- *
- * @param byteVal
- * the byteValue to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public void write(final int byteVal) throws IOException {
- write(new byte[] { (byte) (byteVal & 0xFF) });
- }
-
- /**
- * Writes length bytes from the specified byte array starting at offset to
- * this output stream.
- *
- * @param data
- * the byte array to write.
- * @param offset
- * the start offset in the data.
- * @param length
- * the number of bytes to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public synchronized void write(final byte[] data, int offset, int length)
- throws IOException {
- Preconditions.checkArgument(data != null, "null data");
-
- if (offset < 0 || length < 0 || length > data.length - offset) {
- throw new IndexOutOfBoundsException();
- }
-
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
-
- while (outBuffer.remaining() < length) {
-
- int remaining = outBuffer.remaining();
- outBuffer.put(data, offset, remaining);
-
- // upload payload to azure storage
- addBlockUploadCommand();
-
- offset += remaining;
- length -= remaining;
- }
-
- outBuffer.put(data, offset, length);
- }
-
-
- /**
- * Flushes this output stream and forces any buffered output bytes to be
- * written out. If any data remains in the payload it is committed to the
- * service. Data is queued for writing and forced out to the service
- * before the call returns.
- */
- @Override
- public void flush() throws IOException {
-
- if (closed) {
- // calling close() after the stream is closed starts with call to flush()
- return;
- }
-
- addBlockUploadCommand();
-
- if (committedBlobLength.get() < blobLength) {
- try {
- // wait until the block list is committed
- addFlushCommand().await();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete.
- */
- @Override
- public void hsync() throws IOException {
- // when block compaction is disabled, hsync is empty function
- if (compactionEnabled) {
- flush();
- }
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete.
- */
- @Override
- public void hflush() throws IOException {
- // when block compaction is disabled, hflush is empty function
- if (compactionEnabled) {
- flush();
- }
- }
-
- /**
- * The Synchronization capabilities of this stream depend upon the compaction
- * policy.
- * @param capability string to query the stream support for.
- * @return true for hsync and hflush when compaction is enabled.
- */
- @Override
- public boolean hasCapability(String capability) {
- if (!compactionEnabled) {
- return false;
- }
- return StoreImplementationUtils.isProbeForSyncable(capability);
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete. Close the access to the stream and
- * shutdown the upload thread pool.
- * If the blob was created, its lease will be released.
- * Any error encountered caught in threads and stored will be rethrown here
- * after cleanup.
- */
- @Override
- public synchronized void close() throws IOException {
-
- LOG.debug("close {} ", key);
-
- if (closed) {
- return;
- }
-
- // Upload the last block regardless of compactionEnabled flag
- flush();
-
- // Initiates an orderly shutdown in which previously submitted tasks are
- // executed.
- ioThreadPool.shutdown();
-
- try {
- // wait up to CLOSE_UPLOAD_DELAY minutes to upload all the blocks
- if (!ioThreadPool.awaitTermination(CLOSE_UPLOAD_DELAY, TimeUnit.MINUTES)) {
- LOG.error("Time out occurred while close() is waiting for IO request to"
- + " finish in append"
- + " for blob : {}",
- key);
- NativeAzureFileSystemHelper.logAllLiveStackTraces();
- throw new AzureException("Timed out waiting for IO requests to finish");
- }
- } catch(InterruptedException ex) {
- Thread.currentThread().interrupt();
- }
-
- // release the lease
- if (firstError.get() == null && blobExist) {
- try {
- lease.free();
- } catch (StorageException ex) {
- LOG.debug("Lease free update blob {} encountered Storage Exception:"
- + " {} Error Code : {}",
- key,
- ex,
- ex.getErrorCode());
- maybeSetFirstError(new AzureException(ex));
- }
- }
-
- closed = true;
-
- // finally, throw the first exception raised if it has not
- // been thrown elsewhere.
- if (firstError.get() != null && !firstErrorThrown) {
- throw firstError.get();
- }
- }
-
- /**
- * Helper method used to generate the blockIDs. The algorithm used is similar
- * to the Azure storage SDK.
- */
- private void setBlocksCountAndBlockIdPrefix(List<BlockEntry> blockEntries) {
-
- if (nextBlockCount == UNSET_BLOCKS_COUNT && blockIdPrefix == null) {
-
- Random sequenceGenerator = new Random();
-
- String blockZeroBlockId = (!blockEntries.isEmpty())
- ? blockEntries.get(0).getId()
- : "";
- String prefix = UUID.randomUUID().toString() + "-";
- String sampleNewerVersionBlockId = generateNewerVersionBlockId(prefix,
- 0);
-
- if (!blockEntries.isEmpty()
- && blockZeroBlockId.length() < sampleNewerVersionBlockId.length()) {
-
- // If blob has already been created with 2.2.0, append subsequent blocks
- // with older version (2.2.0) blockId compute nextBlockCount, the way it
- // was done before; and don't use blockIdPrefix
- this.blockIdPrefix = "";
- nextBlockCount = (long) (sequenceGenerator.nextInt(Integer.MAX_VALUE))
- + sequenceGenerator.nextInt(
- Integer.MAX_VALUE - MAX_BLOCK_COUNT);
- nextBlockCount += blockEntries.size();
-
- } else {
-
- // If there are no existing blocks, create the first block with newer
- // version (4.2.0) blockId. If blob has already been created with 4.2.0,
- // append subsequent blocks with newer version (4.2.0) blockId
- this.blockIdPrefix = prefix;
- nextBlockCount = blockEntries.size();
- }
- }
- }
-
- /**
- * Helper method that generates the next block id for uploading a block to
- * azure storage.
- * @return String representing the block ID generated.
- * @throws IOException if the stream is in invalid state
- */
- private String generateBlockId() throws IOException {
-
- if (nextBlockCount == UNSET_BLOCKS_COUNT || blockIdPrefix == null) {
- throw new AzureException(
- "Append Stream in invalid state. nextBlockCount not set correctly");
- }
-
- return (!blockIdPrefix.isEmpty())
- ? generateNewerVersionBlockId(blockIdPrefix, nextBlockCount++)
- : generateOlderVersionBlockId(nextBlockCount++);
- }
-
- /**
- * Helper method that generates an older (2.2.0) version blockId.
- * @return String representing the block ID generated.
- */
- private String generateOlderVersionBlockId(long id) {
-
- byte[] blockIdInBytes = new byte[8];
- for (int m = 0; m < 8; m++) {
- blockIdInBytes[7 - m] = (byte) ((id >> (8 * m)) & 0xFF);
- }
-
- return new String(
- Base64.encodeBase64(blockIdInBytes),
- StandardCharsets.UTF_8);
- }
-
- /**
- * Helper method that generates an newer (4.2.0) version blockId.
- * @return String representing the block ID generated.
- */
- private String generateNewerVersionBlockId(String prefix, long id) {
-
- String blockIdSuffix = String.format("%06d", id);
- byte[] blockIdInBytes =
- (prefix + blockIdSuffix).getBytes(StandardCharsets.UTF_8);
- return new String(Base64.encodeBase64(blockIdInBytes), StandardCharsets.UTF_8);
- }
-
- /**
- * This is shared between upload block Runnable and CommitBlockList. The
- * method captures retry logic
- * @param blockId block name
- * @param dataPayload block content
- */
- private void writeBlockRequestInternal(String blockId,
- ByteBuffer dataPayload,
- boolean bufferPoolBuffer) {
- IOException lastLocalException = null;
-
- int uploadRetryAttempts = 0;
- while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
- try {
- long startTime = System.nanoTime();
-
- blob.uploadBlock(blockId, accessCondition, new ByteArrayInputStream(
- dataPayload.array()), dataPayload.position(),
- new BlobRequestOptions(), opContext);
-
- LOG.debug("upload block finished for {} ms. block {} ",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startTime), blockId);
- break;
-
- } catch(Exception ioe) {
- LOG.debug("Encountered exception during uploading block for Blob {}"
- + " Exception : {}", key, ioe);
- uploadRetryAttempts++;
- lastLocalException = new AzureException(
- "Encountered Exception while uploading block: " + ioe, ioe);
- try {
- Thread.sleep(
- BLOCK_UPLOAD_RETRY_INTERVAL * (uploadRetryAttempts + 1));
- } catch(InterruptedException ie) {
- Thread.currentThread().interrupt();
- break;
- }
- }
- }
-
- if (bufferPoolBuffer) {
- poolReadyByteBuffers.putBuffer(dataPayload);
- }
-
- if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
- maybeSetFirstError(lastLocalException);
- }
- }
-
- /**
- * Set {@link #firstError} to the exception if it is not already set.
- * @param exception exception to save
- */
- private void maybeSetFirstError(IOException exception) {
- firstError.compareAndSet(null, exception);
- }
-
-
- /**
- * Throw the first error caught if it has not been raised already
- * @throws IOException if one is caught and needs to be thrown.
- */
- private void maybeThrowFirstError() throws IOException {
- if (firstError.get() != null) {
- firstErrorThrown = true;
- throw firstError.get();
- }
- }
-
- /**
- * Write block list. The method captures retry logic
- */
- private void writeBlockListRequestInternal() {
-
- IOException lastLocalException = null;
-
- int uploadRetryAttempts = 0;
- while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
- try {
-
- long startTime = System.nanoTime();
-
- blob.commitBlockList(blockEntries, accessCondition,
- new BlobRequestOptions(), opContext);
-
- LOG.debug("Upload block list took {} ms for blob {} ",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startTime), key);
- break;
-
- } catch(Exception ioe) {
- LOG.debug("Encountered exception during uploading block for Blob {}"
- + " Exception : {}", key, ioe);
- uploadRetryAttempts++;
- lastLocalException = new AzureException(
- "Encountered Exception while uploading block: " + ioe, ioe);
- try {
- Thread.sleep(
- BLOCK_UPLOAD_RETRY_INTERVAL * (uploadRetryAttempts + 1));
- } catch(InterruptedException ie) {
- Thread.currentThread().interrupt();
- break;
- }
- }
- }
-
- if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
- maybeSetFirstError(lastLocalException);
- }
- }
-
- /**
- * A ThreadFactory that creates uploader thread with
- * meaningful names helpful for debugging purposes.
- */
- class UploaderThreadFactory implements ThreadFactory {
-
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new SubjectInheritingThread(r);
- t.setName(String.format("%s-%d", THREAD_ID_PREFIX,
- threadSequenceNumber.getAndIncrement()));
- return t;
- }
- }
-
- /**
- * Upload block commands.
- */
- private class UploadBlockCommand extends UploadCommand {
-
- // the block content for upload
- private final ByteBuffer payload;
-
- // description of the block
- private final BlockEntry entry;
-
- UploadBlockCommand(String blockId, ByteBuffer payload) {
-
- super(blobLength);
-
- BlockEntry blockEntry = new BlockEntry(blockId);
- blockEntry.setSize(payload.position());
- blockEntry.setSearchMode(BlockSearchMode.LATEST);
-
- this.payload = payload;
- this.entry = blockEntry;
-
- uncommittedBlockEntries.add(blockEntry);
- }
-
- /**
- * Execute command.
- */
- void execute() throws InterruptedException {
-
- uploadingSemaphore.acquire(1);
- writeBlockRequestInternal(entry.getId(), payload, true);
- uploadingSemaphore.release(1);
-
- }
-
- void dump() {
- LOG.debug("upload block {} size: {} for blob {}",
- entry.getId(),
- entry.getSize(),
- key);
- }
- }
-
- /**
- * Upload blob block list commands.
- */
- private class UploadBlockListCommand extends UploadCommand {
-
- private BlockEntry lastBlock = null;
-
- UploadBlockListCommand() {
- super(blobLength);
-
- if (!uncommittedBlockEntries.isEmpty()) {
- lastBlock = uncommittedBlockEntries.getLast();
- }
- }
-
- void awaitAsDependent() throws InterruptedException {
- // empty. later commit block does not need to wait previous commit block
- // lists.
- }
-
- void dump() {
- LOG.debug("commit block list with {} blocks for blob {}",
- uncommittedBlockEntries.size(), key);
- }
-
- /**
- * Execute command.
- */
- public void execute() throws InterruptedException, IOException {
-
- if (committedBlobLength.get() >= getCommandBlobOffset()) {
- LOG.debug("commit already applied for {}", key);
- return;
- }
-
- if (lastBlock == null) {
- LOG.debug("nothing to commit for {}", key);
- return;
- }
-
- LOG.debug("active commands: {} for {}", activeBlockCommands.size(), key);
-
- for (UploadCommand activeCommand : activeBlockCommands) {
- if (activeCommand.getCommandBlobOffset() < getCommandBlobOffset()) {
- activeCommand.dump();
- activeCommand.awaitAsDependent();
- } else {
- break;
- }
- }
-
- // stop all uploads until the block list is committed
- uploadingSemaphore.acquire(MAX_NUMBER_THREADS_IN_THREAD_POOL);
-
- BlockEntry uncommittedBlock;
- do {
- uncommittedBlock = uncommittedBlockEntries.poll();
- blockEntries.add(uncommittedBlock);
- } while (uncommittedBlock != lastBlock);
-
- if (blockEntries.size() > activateCompactionBlockCount) {
- LOG.debug("Block compaction: activated with {} blocks for {}",
- blockEntries.size(), key);
-
- // Block compaction
- long startCompaction = System.nanoTime();
- blockCompaction();
- LOG.debug("Block compaction finished for {} ms with {} blocks for {}",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startCompaction),
- blockEntries.size(), key);
- }
-
- writeBlockListRequestInternal();
-
- uploadingSemaphore.release(MAX_NUMBER_THREADS_IN_THREAD_POOL);
-
- // remove blocks previous commands
- for (Iterator<UploadCommand> it = activeBlockCommands.iterator();
- it.hasNext();) {
- UploadCommand activeCommand = it.next();
- if (activeCommand.getCommandBlobOffset() <= getCommandBlobOffset()) {
- it.remove();
- } else {
- break;
- }
- }
-
- committedBlobLength.set(getCommandBlobOffset());
- }
-
- /**
- * Internal output stream with read access to the internal buffer.
- */
- private class ByteArrayOutputStreamInternal extends ByteArrayOutputStream {
-
- ByteArrayOutputStreamInternal(int size) {
- super(size);
- }
-
- byte[] getByteArray() {
- return buf;
- }
- }
-
- /**
- * Block compaction process.
- *
- * Block compaction is only enabled when the number of blocks exceeds
- * activateCompactionBlockCount. The algorithm searches for the longest
- * segment [b..e) where (e-b) > 2 && |b| + |b+1| ... |e-1| < maxBlockSize
- * such that size(b1) + size(b2) + ... + size(bn) < maximum-block-size.
- * It then downloads the blocks in the sequence, concatenates the data to
- * form a single block, uploads this new block, and updates the block
- * list to replace the sequence of blocks with the new block.
- */
- private void blockCompaction() throws IOException {
- //current segment [segmentBegin, segmentEnd) and file offset/size of the
- // current segment
- int segmentBegin = 0, segmentEnd = 0;
- long segmentOffsetBegin = 0, segmentOffsetEnd = 0;
-
- //longest segment [maxSegmentBegin, maxSegmentEnd) and file offset/size of
- // the longest segment
- int maxSegmentBegin = 0, maxSegmentEnd = 0;
- long maxSegmentOffsetBegin = 0, maxSegmentOffsetEnd = 0;
-
- for (BlockEntry block : blockEntries) {
- segmentEnd++;
- segmentOffsetEnd += block.getSize();
- if (segmentOffsetEnd - segmentOffsetBegin > maxBlockSize.get()) {
- if (segmentEnd - segmentBegin > 2) {
- if (maxSegmentEnd - maxSegmentBegin < segmentEnd - segmentBegin) {
- maxSegmentBegin = segmentBegin;
- maxSegmentEnd = segmentEnd;
- maxSegmentOffsetBegin = segmentOffsetBegin;
- maxSegmentOffsetEnd = segmentOffsetEnd - block.getSize();
- }
- }
- segmentBegin = segmentEnd - 1;
- segmentOffsetBegin = segmentOffsetEnd - block.getSize();
- }
- }
-
- if (maxSegmentEnd - maxSegmentBegin > 1) {
-
- LOG.debug("Block compaction: {} blocks for {}",
- maxSegmentEnd - maxSegmentBegin, key);
-
- // download synchronously all the blocks from the azure storage
- ByteArrayOutputStreamInternal blockOutputStream
- = new ByteArrayOutputStreamInternal(maxBlockSize.get());
-
- try {
- long length = maxSegmentOffsetEnd - maxSegmentOffsetBegin;
- blob.downloadRange(maxSegmentOffsetBegin, length, blockOutputStream,
- new BlobRequestOptions(), opContext);
- } catch(StorageException ex) {
- LOG.error(
- "Storage exception encountered during block compaction phase"
- + " : {} Storage Exception : {} Error Code: {}",
- key, ex, ex.getErrorCode());
- throw new AzureException(
- "Encountered Exception while committing append blocks " + ex, ex);
- }
-
- // upload synchronously new block to the azure storage
- String blockId = generateBlockId();
-
- ByteBuffer byteBuffer = ByteBuffer.wrap(
- blockOutputStream.getByteArray());
- byteBuffer.position(blockOutputStream.size());
-
- writeBlockRequestInternal(blockId, byteBuffer, false);
-
- // replace blocks from the longest segment with new block id
- blockEntries.subList(maxSegmentBegin + 1, maxSegmentEnd - 1).clear();
- BlockEntry newBlock = blockEntries.get(maxSegmentBegin);
- newBlock.setId(blockId);
- newBlock.setSearchMode(BlockSearchMode.LATEST);
- newBlock.setSize(maxSegmentOffsetEnd - maxSegmentOffsetBegin);
- }
- }
- }
-
- /**
- * Prepare block upload command and queue the command in thread pool executor.
- */
- private synchronized void addBlockUploadCommand() throws IOException {
-
- maybeThrowFirstError();
-
- if (blobExist && lease.isFreed()) {
- throw new AzureException(String.format(
- "Attempting to upload a block on blob : %s "
- + " that does not have lease on the Blob. Failing upload", key));
- }
-
- int blockSize = outBuffer.position();
- if (blockSize > 0) {
- UploadCommand command = new UploadBlockCommand(generateBlockId(),
- outBuffer);
- activeBlockCommands.add(command);
-
- blobLength += blockSize;
- outBuffer = poolReadyByteBuffers.getBuffer(false, maxBlockSize.get());
-
- ioThreadPool.execute(new WriteRequest(command));
-
- }
- }
-
- /**
- * Prepare block list commit command and queue the command in thread pool
- * executor.
- */
- private synchronized UploadCommand addFlushCommand() throws IOException {
-
- maybeThrowFirstError();
-
- if (blobExist && lease.isFreed()) {
- throw new AzureException(
- String.format("Attempting to upload block list on blob : %s"
- + " that does not have lease on the Blob. Failing upload", key));
- }
-
- UploadCommand command = new UploadBlockListCommand();
- activeBlockCommands.add(command);
-
- ioThreadPool.execute(new WriteRequest(command));
-
- return command;
- }
-
- /**
- * Runnable instance that uploads the block of data to azure storage.
- */
- private class WriteRequest implements Runnable {
- private final UploadCommand command;
-
- WriteRequest(UploadCommand command) {
- this.command = command;
- }
-
- @Override
- public void run() {
-
- try {
- command.dump();
- long startTime = System.nanoTime();
- command.execute();
- command.setCompleted();
- LOG.debug("command finished for {} ms",
- TimeUnit.NANOSECONDS.toMillis(
- System.nanoTime() - startTime));
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- LOG.debug(
- "Encountered exception during execution of command for Blob :"
- + " {} Exception : {}", key, ex);
- firstError.compareAndSet(null, new AzureException(ex));
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java
deleted file mode 100644
index 02ee8d6eac6..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobInputStream.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-
-/**
- * Encapsulates the BlobInputStream used by block blobs and adds support for
- * random access and seek. Random access performance is improved by several
- * orders of magnitude.
- */
-final class BlockBlobInputStream extends FSInputStream {
- private final CloudBlockBlobWrapper blob;
- private final BlobRequestOptions options;
- private final OperationContext opContext;
- private final boolean bufferedPreadDisabled;
- private InputStream blobInputStream = null;
- private int minimumReadSizeInBytes = 0;
- private long streamPositionAfterLastRead = -1;
- // position of next network read within stream
- private long streamPosition = 0;
- // length of stream
- private long streamLength = 0;
- private boolean closed = false;
- // internal buffer, re-used for performance optimization
- private byte[] streamBuffer;
- // zero-based offset within streamBuffer of current read position
- private int streamBufferPosition;
- // length of data written to streamBuffer, streamBuffer may be larger
- private int streamBufferLength;
-
- /**
- * Creates a seek-able stream for reading from block blobs.
- * @param blob a block blob reference.
- * @param options the blob request options.
- * @param opContext the blob operation context.
- * @throws IOException IO failure
- */
- BlockBlobInputStream(CloudBlockBlobWrapper blob, BlobRequestOptions options,
- OperationContext opContext, boolean bufferedPreadDisabled)
- throws IOException {
- this.blob = blob;
- this.options = options;
- this.opContext = opContext;
- this.bufferedPreadDisabled = bufferedPreadDisabled;
-
- this.minimumReadSizeInBytes = blob.getStreamMinimumReadSizeInBytes();
-
- try {
- this.blobInputStream = blob.openInputStream(options, opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
-
- this.streamLength = blob.getProperties().getLength();
- }
-
- private void checkState() throws IOException {
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
- }
-
- /**
- * Reset the internal stream buffer but do not release the memory.
- * The buffer can be reused to avoid frequent memory allocations of
- * a large buffer.
- */
- private void resetStreamBuffer() {
- streamBufferPosition = 0;
- streamBufferLength = 0;
- }
-
- /**
- * Gets the read position of the stream.
- * @return the zero-based byte offset of the read position.
- * @throws IOException IO failure
- */
- @Override
- public synchronized long getPos() throws IOException {
- checkState();
- return (streamBuffer != null)
- ? streamPosition - streamBufferLength + streamBufferPosition
- : streamPosition;
- }
-
- /**
- * Sets the read position of the stream.
- * @param pos a zero-based byte offset in the stream.
- * @throws EOFException if read is out of range
- */
- @Override
- public synchronized void seek(long pos) throws IOException {
- checkState();
- if (pos < 0) {
- throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK + " " + pos);
- }
- if (pos > streamLength) {
- throw new EOFException(
- FSExceptionMessages.CANNOT_SEEK_PAST_EOF + " " + pos);
- }
-
- // calculate offset between the target and current position in the stream
- long offset = pos - getPos();
-
- if (offset == 0) {
- // no=op, no state change
- return;
- }
-
- if (offset > 0) {
- // forward seek, data can be skipped as an optimization
- if (skip(offset) != offset) {
- throw new EOFException(FSExceptionMessages.EOF_IN_READ_FULLY);
- }
- return;
- }
-
- // reverse seek, offset is negative
- if (streamBuffer != null) {
- if (streamBufferPosition + offset >= 0) {
- // target position is inside the stream buffer,
- // only need to move backwards within the stream buffer
- streamBufferPosition += offset;
- } else {
- // target position is outside the stream buffer,
- // need to reset stream buffer and move position for next network read
- resetStreamBuffer();
- streamPosition = pos;
- }
- } else {
- streamPosition = pos;
- }
-
- // close BlobInputStream after seek is invoked because BlobInputStream
- // does not support seek
- closeBlobInputStream();
- }
-
- /**
- * Seeks an secondary copy of the data. This method is not supported.
- * @param targetPos a zero-based byte offset in the stream.
- * @return false
- * @throws IOException IO failure
- */
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- /**
- * Gets the number of bytes that can be read (or skipped over) without
- * performing a network operation.
- * @throws IOException IO failure
- */
- @Override
- public synchronized int available() throws IOException {
- checkState();
- if (blobInputStream != null) {
- return blobInputStream.available();
- } else {
- return (streamBuffer == null)
- ? 0
- : streamBufferLength - streamBufferPosition;
- }
- }
-
- private void closeBlobInputStream() throws IOException {
- if (blobInputStream != null) {
- try {
- blobInputStream.close();
- } finally {
- blobInputStream = null;
- }
- }
- }
-
- /**
- * Closes this stream and releases any system resources associated with it.
- * @throws IOException IO failure
- */
- @Override
- public synchronized void close() throws IOException {
- closed = true;
- closeBlobInputStream();
- streamBuffer = null;
- streamBufferPosition = 0;
- streamBufferLength = 0;
- }
-
- private int doNetworkRead(byte[] buffer, int offset, int len)
- throws IOException {
- MemoryOutputStream outputStream;
- boolean needToCopy = false;
-
- if (streamPositionAfterLastRead == streamPosition) {
- // caller is reading sequentially, so initialize the stream buffer
- if (streamBuffer == null) {
- streamBuffer = new byte[(int) Math.min(minimumReadSizeInBytes,
- streamLength)];
- }
- resetStreamBuffer();
- outputStream = new MemoryOutputStream(streamBuffer, streamBufferPosition,
- streamBuffer.length);
- needToCopy = true;
- } else {
- outputStream = new MemoryOutputStream(buffer, offset, len);
- }
-
- long bytesToRead = Math.min(
- minimumReadSizeInBytes,
- Math.min(
- outputStream.capacity(),
- streamLength - streamPosition));
-
- try {
- blob.downloadRange(streamPosition, bytesToRead, outputStream, options,
- opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
-
- int bytesRead = outputStream.size();
- if (bytesRead > 0) {
- streamPosition += bytesRead;
- streamPositionAfterLastRead = streamPosition;
- int count = Math.min(bytesRead, len);
- if (needToCopy) {
- streamBufferLength = bytesRead;
- System.arraycopy(streamBuffer, streamBufferPosition, buffer, offset,
- count);
- streamBufferPosition += count;
- }
- return count;
- } else {
- // This may happen if the blob was modified after the length was obtained.
- throw new EOFException("End of stream reached unexpectedly.");
- }
- }
-
- @Override
- public int read(long position, byte[] buffer, int offset, int length)
- throws IOException {
- synchronized (this) {
- checkState();
- }
- if (!bufferedPreadDisabled) {
- // This will do a seek + read in which the streamBuffer will get used.
- return super.read(position, buffer, offset, length);
- }
- validatePositionedReadArgs(position, buffer, offset, length);
- if (length == 0) {
- return 0;
- }
- if (position >= streamLength) {
- throw new EOFException("position is beyond stream capacity");
- }
- MemoryOutputStream os = new MemoryOutputStream(buffer, offset, length);
- long bytesToRead = Math.min(minimumReadSizeInBytes,
- Math.min(os.capacity(), streamLength - position));
- try {
- blob.downloadRange(position, bytesToRead, os, options, opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
- int bytesRead = os.size();
- if (bytesRead == 0) {
- // This may happen if the blob was modified after the length was obtained.
- throw new EOFException("End of stream reached unexpectedly.");
- }
- return bytesRead;
- }
-
- /**
- * Reads up to <code>len</code> bytes of data from the input stream into an
- * array of bytes.
- * @param b a buffer into which the data is written.
- * @param offset a start offset into {@code buffer} where the data is written.
- * @param len the maximum number of bytes to be read.
- * @return the number of bytes written into {@code buffer}, or -1.
- * @throws IOException IO failure
- */
- @Override
- public synchronized int read(byte[] b, int offset, int len)
- throws IOException {
- checkState();
- NativeAzureFileSystemHelper.validateReadArgs(b, offset, len);
- if (blobInputStream != null) {
- int numberOfBytesRead = blobInputStream.read(b, offset, len);
- streamPosition += numberOfBytesRead;
- return numberOfBytesRead;
- } else {
- if (offset < 0 || len < 0 || len > b.length - offset) {
- throw new IndexOutOfBoundsException("read arguments out of range");
- }
- if (len == 0) {
- return 0;
- }
-
- int bytesRead = 0;
- int available = available();
- if (available > 0) {
- bytesRead = Math.min(available, len);
- System.arraycopy(streamBuffer, streamBufferPosition, b, offset,
- bytesRead);
- streamBufferPosition += bytesRead;
- }
-
- if (len == bytesRead) {
- return len;
- }
- if (streamPosition >= streamLength) {
- return (bytesRead > 0) ? bytesRead : -1;
- }
-
- offset += bytesRead;
- len -= bytesRead;
-
- return bytesRead + doNetworkRead(b, offset, len);
- }
- }
-
- /**
- * Reads the next byte of data from the stream.
- * @return the next byte of data, or -1
- * @throws IOException IO failure
- */
- @Override
- public int read() throws IOException {
- byte[] buffer = new byte[1];
- int numberOfBytesRead = read(buffer, 0, 1);
- return (numberOfBytesRead < 1) ? -1 : buffer[0];
- }
-
- /**
- * Skips over and discards n bytes of data from this input stream.
- * @param n the number of bytes to be skipped.
- * @return the actual number of bytes skipped.
- * @throws IOException IO failure
- * @throws IndexOutOfBoundsException if n is negative or if the sum of n
- * and the current value of getPos() is greater than the length of the stream.
- */
- @Override
- public synchronized long skip(long n) throws IOException {
- checkState();
-
- if (blobInputStream != null) {
- // blobInput stream is open; delegate the work to it
- long skipped = blobInputStream.skip(n);
- // update position to the actual skip value
- streamPosition += skipped;
- return skipped;
- }
-
- // no blob stream; implement the skip logic directly
- if (n < 0 || n > streamLength - getPos()) {
- throw new IndexOutOfBoundsException("skip range");
- }
-
- if (streamBuffer != null) {
- // there's a buffer, so seek with it
- if (n < streamBufferLength - streamBufferPosition) {
- // new range is in the buffer, so just update the buffer position
- // skip within the buffer.
- streamBufferPosition += (int) n;
- } else {
- // skip is out of range, so move position to ne value and reset
- // the buffer ready for the next read()
- streamPosition = getPos() + n;
- resetStreamBuffer();
- }
- } else {
- // no stream buffer; increment the stream position ready for
- // the next triggered connection & read
- streamPosition += n;
- }
- return n;
- }
-
- /**
- * An <code>OutputStream</code> backed by a user-supplied buffer.
- */
- static class MemoryOutputStream extends OutputStream {
- private final byte[] buffer;
- private final int offset;
- private final int length;
- private int writePosition;
-
- /**
- * Creates a <code>MemoryOutputStream</code> from a user-supplied buffer.
- * @param buffer an array of bytes.
- * @param offset a starting offset in <code>buffer</code> where the data
- * will be written.
- * @param length the maximum number of bytes to be written to the stream.
- */
- MemoryOutputStream(byte[] buffer, int offset, int length) {
- if (buffer == null) {
- throw new NullPointerException("buffer");
- }
- if (offset < 0 || length < 0 || length > buffer.length - offset) {
- throw new IndexOutOfBoundsException("offset out of range of buffer");
- }
- this.buffer = buffer;
- this.offset = offset;
- this.length = length;
- this.writePosition = offset;
- }
-
- /**
- * Gets the current size of the stream.
- */
- public synchronized int size() {
- return writePosition - offset;
- }
-
- /**
- * Gets the current capacity of the stream.
- */
- public synchronized int capacity() {
- return length;
- }
-
- /**
- * Writes the next byte to the stream.
- * @param b the byte to be written.
- * @throws IOException IO failure
- */
- public synchronized void write(int b) throws IOException {
- if (size() > length - 1) {
- throw new IOException("No space for more writes");
- }
- buffer[writePosition++] = (byte) b;
- }
-
- /**
- * Writes a range of bytes to the stream.
- * @param b a byte array.
- * @param off the start offset in <code>buffer</code> from which the data
- * is read.
- * @param length the number of bytes to be written.
- * @throws IOException IO failure
- */
- public synchronized void write(byte[] b, int off, int length)
- throws IOException {
- if (b == null) {
- throw new NullPointerException("Null buffer argument");
- }
- if (off < 0 || length < 0 || length > b.length - off) {
- throw new IndexOutOfBoundsException("array write offset");
- }
- System.arraycopy(b, off, buffer, writePosition, length);
- writePosition += length;
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java
deleted file mode 100644
index 5fa0ad8ae34..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/CachingAuthorizer.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
-
-/**
- * Class that provides caching for Authorize and getSasUri calls
- * @param <K> - The cache key type
- * @param <V> - The cached value type
- */
-public class CachingAuthorizer<K, V> {
-
- public static final Logger LOG = LoggerFactory
- .getLogger(CachingAuthorizer.class);
-
- private Cache<K, V> cache;
- private boolean isEnabled = false;
- private long cacheEntryExpiryPeriodInMinutes;
- private String label;
-
- public static final String KEY_AUTH_SERVICE_CACHING_ENABLE =
- "fs.azure.authorization.caching.enable";
-
- public static final boolean KEY_AUTH_SERVICE_CACHING_ENABLE_DEFAULT = false;
-
- public static final String KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES =
- "fs.azure.authorization.caching.maxentries";
-
- public static final int KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES_DEFAULT = 512;
-
- public CachingAuthorizer(long ttlInMinutes, String label) {
- cacheEntryExpiryPeriodInMinutes = ttlInMinutes;
- this.label = label;
- if (cacheEntryExpiryPeriodInMinutes <= 0) {
- isEnabled = false;
- }
- }
-
-
- public void init(Configuration conf) {
-
- isEnabled = conf.getBoolean(KEY_AUTH_SERVICE_CACHING_ENABLE, KEY_AUTH_SERVICE_CACHING_ENABLE_DEFAULT);
-
- if (isEnabled) {
- LOG.debug("{} : Initializing CachingAuthorizer instance", label);
- cache = CacheBuilder.newBuilder()
- .maximumSize(
- conf.getInt(
- KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES,
- KEY_AUTH_SERVICE_CACHING_MAX_ENTRIES_DEFAULT
- )
- )
- .expireAfterWrite(cacheEntryExpiryPeriodInMinutes, TimeUnit.MINUTES)
- .build();
- }
- }
-
- /**
- * @param key - Cache key
- * @return null on cache-miss. true/false on cache-hit
- */
- public V get(K key) {
- if (!isEnabled) {
- return null;
- }
-
- V result = cache.getIfPresent(key);
- if (result == null) {
- LOG.debug("{}: CACHE MISS: {}", label, key.toString());
- }
- else {
- LOG.debug("{}: CACHE HIT: {}, {}", label, key.toString(), result.toString());
- }
- return result;
- }
-
- public void put(K key, V value) {
- if (isEnabled) {
- LOG.debug("{}: CACHE PUT: {}, {}", label, key.toString(), value.toString());
- cache.put(key, value);
- }
- }
-
- public void clear() {
- if (isEnabled) {
- cache.invalidateAll();
- }
- }
-}
-
-/**
- * POJO representing the cache key for authorization calls
- */
-class CachedAuthorizerEntry {
-
- private String path;
- private String accessType;
- private String owner;
-
- CachedAuthorizerEntry(String path, String accessType, String owner) {
- this.path = path;
- this.accessType = accessType;
- this.owner = owner;
- }
-
- public String getPath() {
- return path;
- }
-
- public String getAccessType() {
- return accessType;
- }
-
- public String getOwner() {
- return owner;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
-
- if (o == null) {
- return false;
- }
-
- if (!(o instanceof CachedAuthorizerEntry)) {
- return false;
- }
-
- CachedAuthorizerEntry c = (CachedAuthorizerEntry) o;
- return
- this.getPath().equals(c.getPath())
- && this.getAccessType().equals(c.getAccessType())
- && this.getOwner().equals(c.getOwner());
- }
-
- @Override
- public int hashCode() {
- return this.toString().hashCode();
- }
-
- @Override
- public String toString() {
- return path + ":" + accessType + ":" + owner;
- }
-
-}
-
-
-/**
- * POJO representing the cache key for sas-key calls
- */
-class CachedSASKeyEntry {
-
- private String storageAccount;
- private String container;
- private String path;
-
- CachedSASKeyEntry(String storageAccount, String container, String path) {
- this.storageAccount = storageAccount;
- this.container = container;
- this.path = path;
- }
-
- public String getStorageAccount() {
- return storageAccount;
- }
-
- public String getContainer() {
- return container;
- }
-
- public String getPath() {
- return path;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
-
- if (o == null) {
- return false;
- }
-
- if (!(o instanceof CachedSASKeyEntry)) {
- return false;
- }
-
- CachedSASKeyEntry c = (CachedSASKeyEntry) o;
- return
- this.getStorageAccount().equals(c.getStorageAccount())
- && this.getContainer().equals(c.getContainer())
- && this.getPath().equals(c.getPath());
- }
-
- @Override
- public int hashCode() {
- return this.toString().hashCode();
- }
-
- @Override
- public String toString() {
- return storageAccount + ":" + container + ":" + path;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
deleted file mode 100644
index 3c4803cd3df..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.util.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * Throttles storage operations to minimize errors and maximum throughput. This
- * improves throughput by as much as 35% when the service throttles requests due
- * to exceeding account level ingress or egress limits.
- */
-@InterfaceAudience.Private
-class ClientThrottlingAnalyzer {
- private static final Logger LOG = LoggerFactory.getLogger(
- ClientThrottlingAnalyzer.class);
- private static final int DEFAULT_ANALYSIS_PERIOD_MS = 10 * 1000;
- private static final int MIN_ANALYSIS_PERIOD_MS = 1000;
- private static final int MAX_ANALYSIS_PERIOD_MS = 30000;
- private static final double MIN_ACCEPTABLE_ERROR_PERCENTAGE = .1;
- private static final double MAX_EQUILIBRIUM_ERROR_PERCENTAGE = 1;
- private static final double RAPID_SLEEP_DECREASE_FACTOR = .75;
- private static final double RAPID_SLEEP_DECREASE_TRANSITION_PERIOD_MS = 150
- * 1000;
- private static final double SLEEP_DECREASE_FACTOR = .975;
- private static final double SLEEP_INCREASE_FACTOR = 1.05;
- private int analysisPeriodMs;
-
- private volatile int sleepDuration = 0;
- private long consecutiveNoErrorCount = 0;
- private String name = null;
- private Timer timer = null;
- private AtomicReference<BlobOperationMetrics> blobMetrics = null;
-
- private ClientThrottlingAnalyzer() {
- // hide default constructor
- }
-
- /**
- * Creates an instance of the <code>ClientThrottlingAnalyzer</code> class with
- * the specified name.
- *
- * @param name a name used to identify this instance.
- *
- * @throws IllegalArgumentException if name is null or empty.
- */
- ClientThrottlingAnalyzer(String name) throws IllegalArgumentException {
- this(name, DEFAULT_ANALYSIS_PERIOD_MS);
- }
-
- /**
- * Creates an instance of the <code>ClientThrottlingAnalyzer</code> class with
- * the specified name and period.
- *
- * @param name A name used to identify this instance.
- *
- * @param period The frequency, in milliseconds, at which metrics are
- * analyzed.
- *
- * @throws IllegalArgumentException
- * If name is null or empty.
- * If period is less than 1000 or greater than 30000 milliseconds.
- */
- ClientThrottlingAnalyzer(String name, int period)
- throws IllegalArgumentException {
- Preconditions.checkArgument(
- StringUtils.isNotEmpty(name),
- "The argument 'name' cannot be null or empty.");
- Preconditions.checkArgument(
- period >= MIN_ANALYSIS_PERIOD_MS && period <= MAX_ANALYSIS_PERIOD_MS,
- "The argument 'period' must be between 1000 and 30000.");
- this.name = name;
- this.analysisPeriodMs = period;
- this.blobMetrics = new AtomicReference<BlobOperationMetrics>(
- new BlobOperationMetrics(System.currentTimeMillis()));
- this.timer = new Timer(
- String.format("wasb-timer-client-throttling-analyzer-%s", name), true);
- this.timer.schedule(new TimerTaskImpl(),
- analysisPeriodMs,
- analysisPeriodMs);
- }
-
- /**
- * Updates metrics with results from the current storage operation.
- *
- * @param count The count of bytes transferred.
- *
- * @param isFailedOperation True if the operation failed; otherwise false.
- */
- public void addBytesTransferred(long count, boolean isFailedOperation) {
- BlobOperationMetrics metrics = blobMetrics.get();
- if (isFailedOperation) {
- metrics.bytesFailed.addAndGet(count);
- metrics.operationsFailed.incrementAndGet();
- } else {
- metrics.bytesSuccessful.addAndGet(count);
- metrics.operationsSuccessful.incrementAndGet();
- }
- }
-
- /**
- * Suspends the current storage operation, as necessary, to reduce throughput.
- */
- public void suspendIfNecessary() {
- int duration = sleepDuration;
- if (duration > 0) {
- try {
- Thread.sleep(duration);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- @VisibleForTesting
- int getSleepDuration() {
- return sleepDuration;
- }
-
- private int analyzeMetricsAndUpdateSleepDuration(BlobOperationMetrics metrics,
- int sleepDuration) {
- final double percentageConversionFactor = 100;
- double bytesFailed = metrics.bytesFailed.get();
- double bytesSuccessful = metrics.bytesSuccessful.get();
- double operationsFailed = metrics.operationsFailed.get();
- double operationsSuccessful = metrics.operationsSuccessful.get();
- double errorPercentage = (bytesFailed <= 0)
- ? 0
- : percentageConversionFactor
- * bytesFailed
- / (bytesFailed + bytesSuccessful);
- long periodMs = metrics.endTime - metrics.startTime;
-
- double newSleepDuration;
-
- if (errorPercentage < MIN_ACCEPTABLE_ERROR_PERCENTAGE) {
- ++consecutiveNoErrorCount;
- // Decrease sleepDuration in order to increase throughput.
- double reductionFactor =
- (consecutiveNoErrorCount * analysisPeriodMs
- >= RAPID_SLEEP_DECREASE_TRANSITION_PERIOD_MS)
- ? RAPID_SLEEP_DECREASE_FACTOR
- : SLEEP_DECREASE_FACTOR;
-
- newSleepDuration = sleepDuration * reductionFactor;
- } else if (errorPercentage < MAX_EQUILIBRIUM_ERROR_PERCENTAGE) {
- // Do not modify sleepDuration in order to stabilize throughput.
- newSleepDuration = sleepDuration;
- } else {
- // Increase sleepDuration in order to minimize error rate.
- consecutiveNoErrorCount = 0;
-
- // Increase sleep duration in order to reduce throughput and error rate.
- // First, calculate target throughput: bytesSuccessful / periodMs.
- // Next, calculate time required to send *all* data (assuming next period
- // is similar to previous) at the target throughput: (bytesSuccessful
- // + bytesFailed) * periodMs / bytesSuccessful. Next, subtract periodMs to
- // get the total additional delay needed.
- double additionalDelayNeeded = 5 * analysisPeriodMs;
- if (bytesSuccessful > 0) {
- additionalDelayNeeded = (bytesSuccessful + bytesFailed)
- * periodMs
- / bytesSuccessful
- - periodMs;
- }
-
- // amortize the additional delay needed across the estimated number of
- // requests during the next period
- newSleepDuration = additionalDelayNeeded
- / (operationsFailed + operationsSuccessful);
-
- final double maxSleepDuration = analysisPeriodMs;
- final double minSleepDuration = sleepDuration * SLEEP_INCREASE_FACTOR;
-
- // Add 1 ms to avoid rounding down and to decrease proximity to the server
- // side ingress/egress limit. Ensure that the new sleep duration is
- // larger than the current one to more quickly reduce the number of
- // errors. Don't allow the sleep duration to grow unbounded, after a
- // certain point throttling won't help, for example, if there are far too
- // many tasks/containers/nodes no amount of throttling will help.
- newSleepDuration = Math.max(newSleepDuration, minSleepDuration) + 1;
- newSleepDuration = Math.min(newSleepDuration, maxSleepDuration);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format(
- "%5.5s, %10d, %10d, %10d, %10d, %6.2f, %5d, %5d, %5d",
- name,
- (int) bytesFailed,
- (int) bytesSuccessful,
- (int) operationsFailed,
- (int) operationsSuccessful,
- errorPercentage,
- periodMs,
- (int) sleepDuration,
- (int) newSleepDuration));
- }
-
- return (int) newSleepDuration;
- }
-
- /**
- * Timer callback implementation for periodically analyzing metrics.
- */
- class TimerTaskImpl extends TimerTask {
- private AtomicInteger doingWork = new AtomicInteger(0);
-
- /**
- * Periodically analyzes a snapshot of the blob storage metrics and updates
- * the sleepDuration in order to appropriately throttle storage operations.
- */
- @Override
- public void run() {
- boolean doWork = false;
- try {
- doWork = doingWork.compareAndSet(0, 1);
-
- // prevent concurrent execution of this task
- if (!doWork) {
- return;
- }
-
- long now = System.currentTimeMillis();
- if (now - blobMetrics.get().startTime >= analysisPeriodMs) {
- BlobOperationMetrics oldMetrics = blobMetrics.getAndSet(
- new BlobOperationMetrics(now));
- oldMetrics.endTime = now;
- sleepDuration = analyzeMetricsAndUpdateSleepDuration(oldMetrics,
- sleepDuration);
- }
- }
- finally {
- if (doWork) {
- doingWork.set(0);
- }
- }
- }
- }
-
- /**
- * Stores blob operation metrics during each analysis period.
- */
- static class BlobOperationMetrics {
- private AtomicLong bytesFailed;
- private AtomicLong bytesSuccessful;
- private AtomicLong operationsFailed;
- private AtomicLong operationsSuccessful;
- private long endTime;
- private long startTime;
-
- BlobOperationMetrics(long startTime) {
- this.startTime = startTime;
- this.bytesFailed = new AtomicLong();
- this.bytesSuccessful = new AtomicLong();
- this.operationsFailed = new AtomicLong();
- this.operationsSuccessful = new AtomicLong();
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java
deleted file mode 100644
index 9da993bd237..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingIntercept.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import com.microsoft.azure.storage.ErrorReceivingResponseEvent;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RequestResult;
-import com.microsoft.azure.storage.ResponseReceivedEvent;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.StorageEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.net.HttpURLConnection;
-
-/**
- * Throttles Azure Storage read and write operations to achieve maximum
- * throughput by minimizing errors. The errors occur when the account ingress
- * or egress limits are exceeded and the server-side throttles requests.
- * Server-side throttling causes the retry policy to be used, but the retry
- * policy sleeps for long periods of time causing the total ingress or egress
- * throughput to be as much as 35% lower than optimal. The retry policy is also
- * after the fact, in that it applies after a request fails. On the other hand,
- * the client-side throttling implemented here happens before requests are made
- * and sleeps just enough to minimize errors, allowing optimal ingress and/or
- * egress throughput.
- */
-@InterfaceAudience.Private
-final class ClientThrottlingIntercept {
- private static final Logger LOG = LoggerFactory.getLogger(
- ClientThrottlingIntercept.class);
- private static ClientThrottlingIntercept singleton = null;
- private ClientThrottlingAnalyzer readThrottler = null;
- private ClientThrottlingAnalyzer writeThrottler = null;
-
- // Hide default constructor
- private ClientThrottlingIntercept() {
- readThrottler = new ClientThrottlingAnalyzer("read");
- writeThrottler = new ClientThrottlingAnalyzer("write");
- LOG.debug("Client-side throttling is enabled for the WASB file system.");
- }
-
- static synchronized void initializeSingleton() {
- if (singleton == null) {
- singleton = new ClientThrottlingIntercept();
- }
- }
-
- static void hook(OperationContext context) {
- context.getErrorReceivingResponseEventHandler().addListener(
- new ErrorReceivingResponseEventHandler());
- context.getSendingRequestEventHandler().addListener(
- new SendingRequestEventHandler());
- context.getResponseReceivedEventHandler().addListener(
- new ResponseReceivedEventHandler());
- }
-
- private static void updateMetrics(HttpURLConnection conn,
- RequestResult result) {
- BlobOperationDescriptor.OperationType operationType
- = BlobOperationDescriptor.getOperationType(conn);
- int status = result.getStatusCode();
- long contentLength = 0;
- // If the socket is terminated prior to receiving a response, the HTTP
- // status may be 0 or -1. A status less than 200 or greater than or equal
- // to 500 is considered an error.
- boolean isFailedOperation = (status < HttpURLConnection.HTTP_OK
- || status >= java.net.HttpURLConnection.HTTP_INTERNAL_ERROR);
-
- switch (operationType) {
- case AppendBlock:
- case PutBlock:
- case PutPage:
- contentLength = BlobOperationDescriptor.getContentLengthIfKnown(conn,
- operationType);
- if (contentLength > 0) {
- singleton.writeThrottler.addBytesTransferred(contentLength,
- isFailedOperation);
- }
- break;
- case GetBlob:
- contentLength = BlobOperationDescriptor.getContentLengthIfKnown(conn,
- operationType);
- if (contentLength > 0) {
- singleton.readThrottler.addBytesTransferred(contentLength,
- isFailedOperation);
- }
- break;
- default:
- break;
- }
- }
-
- /**
- * Called when a network error occurs before the HTTP status and response
- * headers are received. Client-side throttling uses this to collect metrics.
- *
- * @param event The connection, operation, and request state.
- */
- public static void errorReceivingResponse(ErrorReceivingResponseEvent event) {
- updateMetrics((HttpURLConnection) event.getConnectionObject(),
- event.getRequestResult());
- }
-
- /**
- * Called before the Azure Storage SDK sends a request. Client-side throttling
- * uses this to suspend the request, if necessary, to minimize errors and
- * maximize throughput.
- *
- * @param event The connection, operation, and request state.
- */
- public static void sendingRequest(SendingRequestEvent event) {
- BlobOperationDescriptor.OperationType operationType
- = BlobOperationDescriptor.getOperationType(
- (HttpURLConnection) event.getConnectionObject());
- switch (operationType) {
- case GetBlob:
- singleton.readThrottler.suspendIfNecessary();
- break;
- case AppendBlock:
- case PutBlock:
- case PutPage:
- singleton.writeThrottler.suspendIfNecessary();
- break;
- default:
- break;
- }
- }
-
- /**
- * Called after the Azure Storage SDK receives a response. Client-side
- * throttling uses this to collect metrics.
- *
- * @param event The connection, operation, and request state.
- */
- public static void responseReceived(ResponseReceivedEvent event) {
- updateMetrics((HttpURLConnection) event.getConnectionObject(),
- event.getRequestResult());
- }
-
- /**
- * The ErrorReceivingResponseEvent is fired when the Azure Storage SDK
- * encounters a network error before the HTTP status and response headers are
- * received.
- */
- @InterfaceAudience.Private
- static class ErrorReceivingResponseEventHandler
- extends StorageEvent<ErrorReceivingResponseEvent> {
-
- /**
- * Called when a network error occurs before the HTTP status and response
- * headers are received. Client-side throttling uses this to collect
- * metrics.
- *
- * @param event The connection, operation, and request state.
- */
- @Override
- public void eventOccurred(ErrorReceivingResponseEvent event) {
- singleton.errorReceivingResponse(event);
- }
- }
-
- /**
- * The SendingRequestEvent is fired before the Azure Storage SDK sends a
- * request.
- */
- @InterfaceAudience.Private
- static class SendingRequestEventHandler
- extends StorageEvent<SendingRequestEvent> {
-
- /**
- * Called before the Azure Storage SDK sends a request. Client-side
- * throttling uses this to suspend the request, if necessary, to minimize
- * errors and maximize throughput.
- *
- * @param event The connection, operation, and request state.
- */
- @Override
- public void eventOccurred(SendingRequestEvent event) {
- singleton.sendingRequest(event);
- }
- }
-
- /**
- * The ResponseReceivedEvent is fired after the Azure Storage SDK receives a
- * response.
- */
- @InterfaceAudience.Private
- static class ResponseReceivedEventHandler
- extends StorageEvent<ResponseReceivedEvent> {
-
- /**
- * Called after the Azure Storage SDK receives a response. Client-side
- * throttling uses this
- * to collect metrics.
- *
- * @param event The connection, operation, and request state.
- */
- @Override
- public void eventOccurred(ResponseReceivedEvent event) {
- singleton.responseReceived(event);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
deleted file mode 100644
index cbf3ab96160..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/FileMetadata.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-
-/**
- * <p>
- * Holds basic metadata for a file stored in a {@link NativeFileSystemStore}.
- * </p>
- */
-@InterfaceAudience.Private
-class FileMetadata extends FileStatus {
- // this is not final so that it can be cleared to save memory when not needed.
- private String key;
- private final BlobMaterialization blobMaterialization;
-
- /**
- * Constructs a FileMetadata object for a file.
- *
- * @param key
- * The key (path) to the file.
- * @param length
- * The length in bytes of the file.
- * @param lastModified
- * The last modified date (milliseconds since January 1, 1970 UTC.)
- * @param permissionStatus
- * The permission for the file.
- * @param blockSize
- * The Hadoop file block size.
- */
- public FileMetadata(String key, long length, long lastModified,
- PermissionStatus permissionStatus, final long blockSize) {
- super(length, false, 1, blockSize, lastModified, 0,
- permissionStatus.getPermission(),
- permissionStatus.getUserName(),
- permissionStatus.getGroupName(),
- null);
- this.key = key;
- // Files are never implicit.
- this.blobMaterialization = BlobMaterialization.Explicit;
- }
-
- /**
- * Constructs a FileMetadata object for a directory.
- *
- * @param key
- * The key (path) to the directory.
- * @param lastModified
- * The last modified date (milliseconds since January 1, 1970 UTC.)
- * @param permissionStatus
- * The permission for the directory.
- * @param blobMaterialization
- * Whether this is an implicit (no real blob backing it) or explicit
- * directory.
- * @param blockSize
- * The Hadoop file block size.
- */
- public FileMetadata(String key, long lastModified,
- PermissionStatus permissionStatus, BlobMaterialization blobMaterialization,
- final long blockSize) {
- super(0, true, 1, blockSize, lastModified, 0,
- permissionStatus.getPermission(),
- permissionStatus.getUserName(),
- permissionStatus.getGroupName(),
- null);
- this.key = key;
- this.blobMaterialization = blobMaterialization;
- }
-
- @Override
- public Path getPath() {
- Path p = super.getPath();
- if (p == null) {
- // Don't store this yet to reduce memory usage, as it will
- // stay in the Eden Space and later we will update it
- // with the full canonicalized path.
- p = NativeAzureFileSystem.keyToPath(key);
- }
- return p;
- }
-
- /**
- * Returns the Azure storage key for the file. Used internally by the framework.
- *
- * @return The key for the file.
- */
- public String getKey() {
- return key;
- }
-
- /**
- * Indicates whether this is an implicit directory (no real blob backing it)
- * or an explicit one.
- *
- * @return Implicit if this is an implicit directory, or Explicit if it's an
- * explicit directory or a file.
- */
- public BlobMaterialization getBlobMaterialization() {
- return blobMaterialization;
- }
-
- void removeKey() {
- key = null;
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java
deleted file mode 100644
index ed510f67cad..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProvider.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * The interface that every Azure file system key provider must implement.
- */
-@InterfaceAudience.Private
-public interface KeyProvider {
- /**
- * Key providers must implement this method. Given a list of configuration
- * parameters for the specified Azure storage account, retrieve the plaintext
- * storage account key.
- *
- * @param accountName
- * the storage account name
- * @param conf
- * Hadoop configuration parameters
- * @return the plaintext storage account key
- * @throws KeyProviderException Thrown if there is a problem instantiating a
- * KeyProvider or retrieving a key using a KeyProvider object.
- */
- String getStorageAccountKey(String accountName, Configuration conf)
- throws KeyProviderException;
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java
deleted file mode 100644
index b65b2e6f42f..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/KeyProviderException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Thrown if there is a problem instantiating a KeyProvider or retrieving a key
- * using a KeyProvider object.
- */
-@InterfaceAudience.Private
-public class KeyProviderException extends Exception {
- private static final long serialVersionUID = 1L;
-
- public KeyProviderException(String message) {
- super(message);
- }
-
- public KeyProviderException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public KeyProviderException(Throwable t) {
- super(t);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java
deleted file mode 100644
index b573457cd17..00000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/LocalSASKeyGeneratorImpl.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.StringTokenizer;
-import java.util.TimeZone;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.SharedAccessAccountPermissions;
-import com.microsoft.azure.storage.SharedAccessAccountPolicy;
-import com.microsoft.azure.storage.SharedAccessAccountResourceType;
-import com.microsoft.azure.storage.SharedAccessAccountService;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/***
- * Local SAS Key Generation implementation. This class resides in
- * the same address space as the WASB driver.
- *
- * This class gets typically used for testing purposes.
- *
- */
-
-public class LocalSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
-
- public static final Logger LOG = LoggerFactory.getLogger(
- LocalSASKeyGeneratorImpl.class);
-
- /**
- * Map to cache CloudStorageAccount instances.
- */
- private Map<String, CloudStorageAccount> storageAccountMap;
- private CachingAuthorizer<CachedSASKeyEntry, URI> cache;
- private static final int HOURS_IN_DAY = 24;
-
- public LocalSASKeyGeneratorImpl(Configuration conf) {
- super(conf);
- storageAccountMap = new HashMap<String, CloudStorageAccount>();
- cache = new CachingAuthorizer<>(getSasKeyExpiryPeriod(), "SASKEY");
- cache.init(conf);
- }
-
- /**
- * Implementation to generate SAS Key for a container
- */
- @Override
- public URI getContainerSASUri(String accountName, String container)
- throws SASKeyGenerationException {
-
- LOG.debug("Retrieving Container SAS URI For {}@{}", container, accountName);
- try {
-
- CachedSASKeyEntry cacheKey = new CachedSASKeyEntry(accountName, container, "/");
- URI cacheResult = cache.get(cacheKey);
- if (cacheResult != null) {
- return cacheResult;
- }
-
- CloudStorageAccount account =
- getSASKeyBasedStorageAccountInstance(accountName);
- CloudBlobClient client = account.createCloudBlobClient();
- URI sasKey = client.getCredentials().transformUri(
- client.getContainerReference(container).getUri());
- cache.put(cacheKey, sasKey);
- return sasKey;
-
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while"
- + " generating SAS Key for container " + container + " inside "
- + "storage account " + accountName, stoEx);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException while"
- + " generating SAS Key for container " + container + " inside storage"
- + " account " + accountName, uriSyntaxEx);
- }
- }
-
- /**
- * Helper method that creates a CloudStorageAccount instance based on
- * SAS key for accountName
- *
- * @param accountName Storage Account Name
- * @return CloudStorageAccount instance created using SAS key for
- * the Storage Account.
- * @throws SASKeyGenerationException
- */
- private CloudStorageAccount getSASKeyBasedStorageAccountInstance(
- String accountName) throws SASKeyGenerationException {
- LOG.debug("Creating SAS key from account instance {}", accountName);
- try {
-
- String accountNameWithoutDomain =
- getAccountNameWithoutDomain(accountName);
-
- CloudStorageAccount account =
- getStorageAccountInstance(accountNameWithoutDomain,
- AzureNativeFileSystemStore.getAccountKeyFromConfiguration(
- accountName, getConf()));
-
- return new CloudStorageAccount(
- new StorageCredentialsSharedAccessSignature(
- account.generateSharedAccessSignature(
- getDefaultAccountAccessPolicy())), false,
- account.getEndpointSuffix(), accountNameWithoutDomain);
-
- } catch (KeyProviderException keyProviderEx) {
- throw new SASKeyGenerationException("Encountered KeyProviderException"
- + " while retrieving Storage key from configuration for account "
- + accountName, keyProviderEx);
- } catch (InvalidKeyException invalidKeyEx) {
- throw new SASKeyGenerationException("Encoutered InvalidKeyException "
- + "while generating Account level SAS key for account" + accountName,
- invalidKeyEx);
- } catch(StorageException storeEx) {
- throw new SASKeyGenerationException("Encoutered StorageException while "
- + "generating Account level SAS key for account" + accountName,
- storeEx);
- } catch(URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException for"
- + " account " + accountName, uriSyntaxEx);
- }
- }
-
- /**
- * Implementation for generation of Relative Path Blob SAS Uri.
- */
- @Override
- public URI getRelativeBlobSASUri(String accountName, String container,
- String relativePath) throws SASKeyGenerationException {
-
- CloudBlobContainer sc = null;
- CloudBlobClient client = null;
- CachedSASKeyEntry cacheKey = null;
-
- try {
-
- cacheKey = new CachedSASKeyEntry(accountName, container, relativePath);
- URI cacheResult = cache.get(cacheKey);
- if (cacheResult != null) {
- return cacheResult;
- }
-
- CloudStorageAccount account =
- getSASKeyBasedStorageAccountInstance(accountName);
- client = account.createCloudBlobClient();
- sc = client.getContainerReference(container);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException "
- + "while getting container references for container " + container
- + " inside storage account : " + accountName, uriSyntaxEx);
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while "
- + "getting container references for container " + container
- + " inside storage account : " + accountName, stoEx);
- }
-
- CloudBlockBlob blob = null;
- try {
- blob = sc.getBlockBlobReference(relativePath);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException while "
- + "getting Block Blob references for container " + container
- + " inside storage account : " + accountName, uriSyntaxEx);
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while "
- + "getting Block Blob references for container " + container
- + " inside storage account : " + accountName, stoEx);
- }
-
- try {
- URI sasKey = client.getCredentials().transformUri(blob.getUri());
- cache.put(cacheKey, sasKey);
- return sasKey;
- } catch (StorageException stoEx) {
- throw new SASKeyGenerationException("Encountered StorageException while "
- + "generating SAS key for Blob: " + relativePath + " inside "
- + "container : " + container + " in Storage Account : " + accountName,
- stoEx);
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException "
- + "while generating SAS key for Blob: " + relativePath + " inside "
- + "container: " + container + " in Storage Account : " + accountName,
- uriSyntaxEx);
- }
- }
-
- /**
- * Helper method that creates CloudStorageAccount Instance using the
- * storage account key.
- * @param accountName Name of the storage account
- * @param accountKey Storage Account key
- * @return CloudStorageAccount instance for the storage account.
- * @throws SASKeyGenerationException
- */
- private CloudStorageAccount getStorageAccountInstance(String accountName,
- String accountKey) throws SASKeyGenerationException {
-
- if (!storageAccountMap.containsKey(accountName)) {
- if (accountKey == null || accountKey.isEmpty()) {
- throw new SASKeyGenerationException(
- "No key for Storage account " + accountName);
- }
-
- CloudStorageAccount account = null;
- try {
- account =
- new CloudStorageAccount(new StorageCredentialsAccountAndKey(
- accountName, accountKey));
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException "
- + "for account " + accountName, uriSyntaxEx);
- }
-
- storageAccountMap.put(accountName, account);
- }
-
- return storageAccountMap.get(accountName);
- }
-
- /**
- * Helper method that returns the Storage account name without
- * the domain name suffix.
- * @param fullAccountName Storage account name with domain name suffix
- * @return String
- */
- private String getAccountNameWithoutDomain(String fullAccountName) {
- StringTokenizer tokenizer = new StringTokenizer(fullAccountName, ".");
- return tokenizer.nextToken();
- }
-
- /**
- * Helper method to generate Access Policy for the Storage Account SAS Key
- * @return SharedAccessAccountPolicy
- */
- private SharedAccessAccountPolicy getDefaultAccountAccessPolicy() {
-
- SharedAccessAccountPolicy ap =
- new SharedAccessAccountPolicy();
-
- Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
- cal.setTime(new Date());
- cal.add(Calendar.HOUR, (int) getSasKeyExpiryPeriod() * HOURS_IN_DAY);
-
- ap.setSharedAccessExpiryTime(cal.getTime());
- ap.setPermissions(getDefaultAccoutSASKeyPermissions());
- ap.setResourceTypes(EnumSet.of(SharedAccessAccountResourceType.CONTAINER,
- SharedAccessAccountResourceType.OBJECT));
- ap.setServices(EnumSet.of(SharedAccessAccountService.BLOB));
-
- return ap;
- }
-
- private EnumSet<SharedAccessAccountPermissions> getDefaultAccoutSASKeyPermissions() {
- return EnumSet.of(SharedAccessAccountPermissions.ADD,
- SharedAccessAccountPermissions.CREATE,
- SharedAccessAccountPermissions.DELETE,
- SharedAccessAccountPermissions.LIST,
- SharedAccessAccountPermissions.READ,
- SharedAccessAccountPermissions.UPDATE,
- SharedAccessAccountPermissions.WRITE);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 4e777da8b40..36d732aeefc 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -18,4039 +18,131 @@
package org.apache.hadoop.fs.azure;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.TimeZone;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.Stack;
-import java.util.HashMap;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BufferedFSInputStream;
-import org.apache.hadoop.fs.CommonPathCapabilities;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PositionedReadable;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.fs.azure.security.RemoteWasbDelegationTokenManager;
-import org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager;
-import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
-import org.apache.hadoop.fs.impl.OpenFileParameters;
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.util.LambdaUtils;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Time;
-
-import org.apache.hadoop.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
-import static org.apache.hadoop.fs.azure.NativeAzureFileSystemHelper.*;
-import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.classification.VisibleForTesting;
-import com.microsoft.azure.storage.StorageException;
-
-/**
- * A {@link FileSystem} for reading and writing files stored on <a
- * href="http://store.azure.com/">Windows Azure</a>. This implementation is
- * blob-based and stores files on Azure in their native form so they can be read
- * by other Azure tools.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class NativeAzureFileSystem extends FileSystem {
- private static final int USER_WX_PERMISION = 0300;
- private static final String USER_HOME_DIR_PREFIX_DEFAULT = "/user";
- /**
- * A description of a folder rename operation, including the source and
- * destination keys, and descriptions of the files in the source folder.
- */
-
- public static class FolderRenamePending {
- private SelfRenewingLease folderLease;
- private String srcKey;
- private String dstKey;
- private FileMetadata[] fileMetadata = null; // descriptions of source files
- private ArrayList<String> fileStrings = null;
- private NativeAzureFileSystem fs;
- private static final int MAX_RENAME_PENDING_FILE_SIZE = 10000000;
- private static final int FORMATTING_BUFFER = 10000;
- private boolean committed;
- public static final String SUFFIX = "-RenamePending.json";
- private static final ObjectReader READER = new ObjectMapper()
- .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
- .readerFor(JsonNode.class);
-
- // Prepare in-memory information needed to do or redo a folder rename.
- public FolderRenamePending(String srcKey, String dstKey, SelfRenewingLease lease,
- NativeAzureFileSystem fs) throws IOException {
- this.srcKey = srcKey;
- this.dstKey = dstKey;
- this.folderLease = lease;
- this.fs = fs;
-
- // List all the files in the folder.
- long start = Time.monotonicNow();
- fileMetadata = fs.getStoreInterface().list(srcKey, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH);
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for rename operation is: {} ms", fileMetadata.length, (end - start));
-
- this.committed = true;
- }
-
- // Prepare in-memory information needed to do or redo folder rename from
- // a -RenamePending.json file read from storage. This constructor is to use during
- // redo processing.
- public FolderRenamePending(Path redoFile, NativeAzureFileSystem fs)
- throws IllegalArgumentException, IOException {
-
- this.fs = fs;
-
- // open redo file
- Path f = redoFile;
- int l;
- byte[] bytes;
- try (FSDataInputStream input = fs.open(f)) {
- bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
- l = input.read(bytes);
- }
- if (l <= 0) {
- // Jira HADOOP-12678 -Handle empty rename pending metadata file during
- // atomic rename in redo path. If during renamepending file is created
- // but not written yet, then this means that rename operation
- // has not started yet. So we should delete rename pending metadata file.
- LOG.error("Deleting empty rename pending file "
- + redoFile + " -- no data available");
- deleteRenamePendingFile(fs, redoFile);
- return;
- }
- if (l == MAX_RENAME_PENDING_FILE_SIZE) {
- throw new IOException(
- "Error reading pending rename file contents -- "
- + "maximum file size exceeded");
- }
- String contents = new String(bytes, 0, l, StandardCharsets.UTF_8);
-
- // parse the JSON
- JsonNode json = null;
- try {
- json = READER.readValue(contents);
- this.committed = true;
- } catch (JsonMappingException e) {
-
- // The -RedoPending.json file is corrupted, so we assume it was
- // not completely written
- // and the redo operation did not commit.
- this.committed = false;
- } catch (JsonParseException e) {
- this.committed = false;
- } catch (IOException e) {
- this.committed = false;
- }
-
- if (!this.committed) {
- LOG.error("Deleting corruped rename pending file {} \n {}",
- redoFile, contents);
-
- // delete the -RenamePending.json file
- deleteRenamePendingFile(fs, redoFile);
- return;
- }
-
- // initialize this object's fields
- ArrayList<String> fileStrList = new ArrayList<String>();
- JsonNode oldFolderName = json.get("OldFolderName");
- JsonNode newFolderName = json.get("NewFolderName");
- if (oldFolderName == null || newFolderName == null) {
- this.committed = false;
- } else {
- this.srcKey = oldFolderName.textValue();
- this.dstKey = newFolderName.textValue();
- if (this.srcKey == null || this.dstKey == null) {
- this.committed = false;
- } else {
- JsonNode fileList = json.get("FileList");
- if (fileList == null) {
- this.committed = false;
- } else {
- for (int i = 0; i < fileList.size(); i++) {
- fileStrList.add(fileList.get(i).textValue());
- }
- }
- }
- }
- this.fileStrings = fileStrList;
- }
-
- public FileMetadata[] getFiles() {
- return fileMetadata;
- }
-
- public SelfRenewingLease getFolderLease() {
- return folderLease;
- }
-
- /**
- * Deletes rename pending metadata file
- * @param fs -- the file system
- * @param redoFile - rename pending metadata file path
- * @throws IOException - If deletion fails
- */
- @VisibleForTesting
- void deleteRenamePendingFile(FileSystem fs, Path redoFile)
- throws IOException {
- try {
- fs.delete(redoFile, false);
- } catch (IOException e) {
- // If the rename metadata was not found then somebody probably
- // raced with us and finished the delete first
- Throwable t = e.getCause();
- if (t != null && t instanceof StorageException
- && "BlobNotFound".equals(((StorageException) t).getErrorCode())) {
- LOG.warn("rename pending file " + redoFile + " is already deleted");
- } else {
- throw e;
- }
- }
- }
-
- /**
- * Write to disk the information needed to redo folder rename,
- * in JSON format. The file name will be
- * {@code wasb://<sourceFolderPrefix>/folderName-RenamePending.json}
- * The file format will be:
- * <pre>{@code
- * {
- * FormatVersion: "1.0",
- * OperationTime: "<YYYY-MM-DD HH:MM:SS.MMM>",
- * OldFolderName: "<key>",
- * NewFolderName: "<key>",
- * FileList: [ <string> , <string> , ... ]
- * }
- *
- * Here's a sample:
- * {
- * FormatVersion: "1.0",
- * OperationUTCTime: "2014-07-01 23:50:35.572",
- * OldFolderName: "user/ehans/folderToRename",
- * NewFolderName: "user/ehans/renamedFolder",
- * FileList: [
- * "innerFile",
- * "innerFile2"
- * ]
- * } }</pre>
- * @param fs file system on which a file is written.
- * @throws IOException Thrown when fail to write file.
- */
- public void writeFile(NativeAzureFileSystem fs) throws IOException {
- Path path = getRenamePendingFilePath();
- LOG.debug("Preparing to write atomic rename state to {}", path.toString());
- OutputStream output = null;
-
- String contents = makeRenamePendingFileContents();
-
- // Write file.
- try {
- output = fs.createInternal(path, FsPermission.getFileDefault(), false, null);
- output.write(contents.getBytes(StandardCharsets.UTF_8));
- } catch (IOException e) {
- throw new IOException("Unable to write RenamePending file for folder rename from "
- + srcKey + " to " + dstKey, e);
- } finally {
- NativeAzureFileSystemHelper.cleanup(LOG, output);
- }
- }
-
- /**
- * Return the contents of the JSON file to represent the operations
- * to be performed for a folder rename.
- *
- * @return JSON string which represents the operation.
- */
- public String makeRenamePendingFileContents() {
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
- sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
- String time = sdf.format(new Date());
-
- // Make file list string
- StringBuilder builder = new StringBuilder();
- builder.append("[\n");
- for (int i = 0; i != fileMetadata.length; i++) {
- if (i > 0) {
- builder.append(",\n");
- }
- builder.append(" ");
- String noPrefix = StringUtils.removeStart(fileMetadata[i].getKey(), srcKey + "/");
-
- // Quote string file names, escaping any possible " characters or other
- // necessary characters in the name.
- builder.append(quote(noPrefix));
- if (builder.length() >=
- MAX_RENAME_PENDING_FILE_SIZE - FORMATTING_BUFFER) {
-
- // Give up now to avoid using too much memory.
- LOG.error("Internal error: Exceeded maximum rename pending file size of {} bytes.",
- MAX_RENAME_PENDING_FILE_SIZE);
-
- // return some bad JSON with an error message to make it human readable
- return "exceeded maximum rename pending file size";
- }
- }
- builder.append("\n ]");
- String fileList = builder.toString();
-
- // Make file contents as a string. Again, quote file names, escaping
- // characters as appropriate.
- String contents = "{\n"
- + " FormatVersion: \"1.0\",\n"
- + " OperationUTCTime: \"" + time + "\",\n"
- + " OldFolderName: " + quote(srcKey) + ",\n"
- + " NewFolderName: " + quote(dstKey) + ",\n"
- + " FileList: " + fileList + "\n"
- + "}\n";
-
- return contents;
- }
-
- /**
- * This is an exact copy of org.codehaus.jettison.json.JSONObject.quote
- * method.
- *
- * Produce a string in double quotes with backslash sequences in all the
- * right places. A backslash will be inserted within </, allowing JSON
- * text to be delivered in HTML. In JSON text, a string cannot contain a
- * control character or an unescaped quote or backslash.
- * @param string A String
- * @return A String correctly formatted for insertion in a JSON text.
- */
- private String quote(String string) {
- if (string == null || string.length() == 0) {
- return "\"\"";
- }
-
- char c = 0;
- int i;
- int len = string.length();
- StringBuilder sb = new StringBuilder(len + 4);
- String t;
-
- sb.append('"');
- for (i = 0; i < len; i += 1) {
- c = string.charAt(i);
- switch (c) {
- case '\\':
- case '"':
- sb.append('\\');
- sb.append(c);
- break;
- case '/':
- sb.append('\\');
- sb.append(c);
- break;
- case '\b':
- sb.append("\\b");
- break;
- case '\t':
- sb.append("\\t");
- break;
- case '\n':
- sb.append("\\n");
- break;
- case '\f':
- sb.append("\\f");
- break;
- case '\r':
- sb.append("\\r");
- break;
- default:
- if (c < ' ') {
- t = "000" + Integer.toHexString(c);
- sb.append("\\u" + t.substring(t.length() - 4));
- } else {
- sb.append(c);
- }
- }
- }
- sb.append('"');
- return sb.toString();
- }
-
- public String getSrcKey() {
- return srcKey;
- }
-
- public String getDstKey() {
- return dstKey;
- }
-
- public FileMetadata getSourceMetadata() throws IOException {
- return fs.getStoreInterface().retrieveMetadata(srcKey);
- }
-
- /**
- * Execute a folder rename. This is the execution path followed
- * when everything is working normally. See redo() for the alternate
- * execution path for the case where we're recovering from a folder rename
- * failure.
- * @throws IOException Thrown when fail to renaming.
- */
- public void execute() throws IOException {
-
- AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
- @Override
- public boolean execute(FileMetadata file) throws IOException{
- renameFile(file);
- return true;
- }
- };
-
- AzureFileSystemThreadPoolExecutor executor = this.fs.getThreadPoolExecutor(this.fs.renameThreadCount,
- "AzureBlobRenameThread", "Rename", getSrcKey(), AZURE_RENAME_THREADS);
-
- executor.executeParallel(this.getFiles(), task);
-
- // Rename the source folder 0-byte root file itself.
- FileMetadata srcMetadata2 = this.getSourceMetadata();
- if (srcMetadata2.getBlobMaterialization() ==
- BlobMaterialization.Explicit) {
-
- // It already has a lease on it from the "prepare" phase so there's no
- // need to get one now. Pass in existing lease to allow file delete.
- fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(),
- false, folderLease);
- }
-
- // Update the last-modified time of the parent folders of both source and
- // destination.
- fs.updateParentFolderLastModifiedTime(srcKey);
- fs.updateParentFolderLastModifiedTime(dstKey);
- }
-
- // Rename a single file
- @VisibleForTesting
- void renameFile(FileMetadata file) throws IOException{
- // Rename all materialized entries under the folder to point to the
- // final destination.
- if (file.getBlobMaterialization() == BlobMaterialization.Explicit) {
- String srcName = file.getKey();
- String suffix = srcName.substring((this.getSrcKey()).length());
- String dstName = this.getDstKey() + suffix;
-
- // Rename gets exclusive access (via a lease) for files
- // designated for atomic rename.
- // The main use case is for HBase write-ahead log (WAL) and data
- // folder processing correctness. See the rename code for details.
- boolean acquireLease = this.fs.getStoreInterface().isAtomicRenameKey(srcName);
- this.fs.getStoreInterface().rename(srcName, dstName, acquireLease, null);
- }
- }
-
- /** Clean up after execution of rename.
- * @throws IOException Thrown when fail to clean up.
- * */
- public void cleanup() throws IOException {
-
- if (fs.getStoreInterface().isAtomicRenameKey(srcKey)) {
-
- // Remove RenamePending file
- fs.delete(getRenamePendingFilePath(), false);
-
- // Freeing source folder lease is not necessary since the source
- // folder file was deleted.
- }
- }
-
- private Path getRenamePendingFilePath() {
- String fileName = srcKey + SUFFIX;
- Path fileNamePath = keyToPath(fileName);
- Path path = fs.makeAbsolute(fileNamePath);
- return path;
- }
-
- /**
- * Recover from a folder rename failure by redoing the intended work,
- * as recorded in the -RenamePending.json file.
- *
- * @throws IOException Thrown when fail to redo.
- */
- public void redo() throws IOException {
-
- if (!committed) {
-
- // Nothing to do. The -RedoPending.json file should have already been
- // deleted.
- return;
- }
-
- // Try to get a lease on source folder to block concurrent access to it.
- // It may fail if the folder is already gone. We don't check if the
- // source exists explicitly because that could recursively trigger redo
- // and give an infinite recursion.
- SelfRenewingLease lease = null;
- boolean sourceFolderGone = false;
- try {
- lease = fs.leaseSourceFolder(srcKey);
- } catch (AzureException e) {
-
- // If the source folder was not found then somebody probably
- // raced with us and finished the rename first, or the
- // first rename failed right before deleting the rename pending
- // file.
- String errorCode = "";
- try {
- StorageException se = (StorageException) e.getCause();
- errorCode = se.getErrorCode();
- } catch (Exception e2) {
- ; // do nothing -- could not get errorCode
- }
- if (errorCode.equals("BlobNotFound")) {
- sourceFolderGone = true;
- } else {
- throw new IOException(
- "Unexpected error when trying to lease source folder name during "
- + "folder rename redo",
- e);
- }
- }
-
- if (!sourceFolderGone) {
- // Make sure the target folder exists.
- Path dst = fullPath(dstKey);
- if (!fs.existsInternal(dst)) {
- fs.mkdirs(dst);
- }
-
- // For each file inside the folder to be renamed,
- // make sure it has been renamed.
- for(String fileName : fileStrings) {
- finishSingleFileRename(fileName);
- }
-
- // Remove the source folder. Don't check explicitly if it exists,
- // to avoid triggering redo recursively.
- try {
- // Rename the source folder 0-byte root file
- // as destination folder 0-byte root file.
- FileMetadata srcMetaData = this.getSourceMetadata();
- if (srcMetaData.getBlobMaterialization() == BlobMaterialization.Explicit) {
- // We already have a lease. So let's just rename the source blob
- // as destination blob under same lease.
- fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(), false, lease);
- }
-
- // Now we can safely delete the source folder.
- fs.getStoreInterface().delete(srcKey, lease);
- } catch (Exception e) {
- LOG.info("Unable to delete source folder during folder rename redo. "
- + "If the source folder is already gone, this is not an error "
- + "condition. Continuing with redo.", e);
- }
-
- // Update the last-modified time of the parent folders of both source
- // and destination.
- fs.updateParentFolderLastModifiedTime(srcKey);
- fs.updateParentFolderLastModifiedTime(dstKey);
- }
-
- // Remove the -RenamePending.json file.
- fs.delete(getRenamePendingFilePath(), false);
- }
-
- // See if the source file is still there, and if it is, rename it.
- private void finishSingleFileRename(String fileName)
- throws IOException {
- Path srcFile = fullPath(srcKey, fileName);
- Path dstFile = fullPath(dstKey, fileName);
- String srcName = fs.pathToKey(srcFile);
- String dstName = fs.pathToKey(dstFile);
- boolean srcExists = fs.getStoreInterface().explicitFileExists(srcName);
- boolean dstExists = fs.getStoreInterface().explicitFileExists(dstName);
- if(srcExists) {
- // Rename gets exclusive access (via a lease) for HBase write-ahead log
- // (WAL) file processing correctness. See the rename code for details.
- fs.getStoreInterface().rename(srcName, dstName, true, null);
- } else if (!srcExists && dstExists) {
- // The rename already finished, so do nothing.
- ;
- } else {
- // HADOOP-14512
- LOG.warn(
- "Attempting to complete rename of file " + srcKey + "/" + fileName
- + " during folder rename redo, and file was not found in source "
- + "or destination " + dstKey + "/" + fileName + ". "
- + "This must mean the rename of this file has already completed");
- }
- }
-
- // Return an absolute path for the specific fileName within the folder
- // specified by folderKey.
- private Path fullPath(String folderKey, String fileName) {
- return new Path(new Path(fs.getUri()), "/" + folderKey + "/" + fileName);
- }
-
- private Path fullPath(String fileKey) {
- return new Path(new Path(fs.getUri()), "/" + fileKey);
- }
- }
-
- private static final String TRAILING_PERIOD_PLACEHOLDER = "[[.]]";
- private static final Pattern TRAILING_PERIOD_PLACEHOLDER_PATTERN =
- Pattern.compile("\\[\\[\\.\\]\\](?=$|/)");
- private static final Pattern TRAILING_PERIOD_PATTERN = Pattern.compile("\\.(?=$|/)");
-
- @Override
- public String getScheme() {
- return "wasb";
- }
-
- /**
- * If fs.azure.override.canonical.service.name is set as true, return URI of
- * the WASB filesystem, otherwise use the default implementation.
- *
- * @return a service string that uniquely identifies this file system
- */
- @Override
- public String getCanonicalServiceName() {
- if (returnUriAsCanonicalServiceName) {
- return getUri().toString();
- }
- return super.getCanonicalServiceName();
- }
-
-
- /**
- * <p>
- * A {@link FileSystem} for reading and writing files stored on <a
- * href="http://store.azure.com/">Windows Azure</a>. This implementation is
- * blob-based and stores files on Azure in their native form so they can be read
- * by other Azure tools. This implementation uses HTTPS for secure network communication.
- * </p>
- */
- public static class Secure extends NativeAzureFileSystem {
- @Override
- public String getScheme() {
- return "wasbs";
- }
- }
-
- public static final Logger LOG = LoggerFactory.getLogger(NativeAzureFileSystem.class);
-
- /**
- * The time span in seconds before which we consider a temp blob to be
- * dangling (not being actively uploaded to) and up for reclamation.
- *
- * So e.g. if this is 60, then any temporary blobs more than a minute old
- * would be considered dangling.
- */
- static final String AZURE_TEMP_EXPIRY_PROPERTY_NAME = "fs.azure.fsck.temp.expiry.seconds";
- private static final int AZURE_TEMP_EXPIRY_DEFAULT = 3600;
- static final String PATH_DELIMITER = Path.SEPARATOR;
- static final String AZURE_TEMP_FOLDER = "_$azuretmpfolder$";
-
- private static final int AZURE_LIST_ALL = -1;
- private static final int AZURE_UNBOUNDED_DEPTH = -1;
-
- /**
- * The configuration property that determines which group owns files created
- * in WASB.
- */
- private static final String AZURE_DEFAULT_GROUP_PROPERTY_NAME = "fs.azure.permissions.supergroup";
- /**
- * The default value for fs.azure.permissions.supergroup. Chosen as the same
- * default as DFS.
- */
- static final String AZURE_DEFAULT_GROUP_DEFAULT = "supergroup";
-
- /**
- * Configuration property used to specify list of users that can perform
- * chown operation when authorization is enabled in WASB.
- */
- public static final String AZURE_CHOWN_USERLIST_PROPERTY_NAME =
- "fs.azure.chown.allowed.userlist";
-
- static final String AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
-
- /**
- * Configuration property used to specify list of daemon users that can
- * perform chmod operation when authorization is enabled in WASB.
- */
- public static final String AZURE_DAEMON_USERLIST_PROPERTY_NAME =
- "fs.azure.daemon.userlist";
-
- static final String AZURE_DAEMON_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
-
- /**
- * Configuration property used to specify list of users that can perform
- * chmod operation when authorization is enabled in WASB.
- */
- public static final String AZURE_CHMOD_USERLIST_PROPERTY_NAME =
- "fs.azure.chmod.allowed.userlist";
-
- static final String AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
-
- static final String AZURE_RINGBUFFER_CAPACITY_PROPERTY_NAME =
- "fs.azure.ring.buffer.capacity";
- static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME =
- "fs.azure.output.stream.buffer.size";
-
- public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics";
-
- /*
- * Property to enable Append API.
- */
- public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support";
-
- /*
- * Property to override canonical service name with filesystem's URI.
- */
- public static final String RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME = "fs.azure.override.canonical.service.name";
-
- /**
- * The configuration property to set number of threads to be used for rename operation.
- */
- public static final String AZURE_RENAME_THREADS = "fs.azure.rename.threads";
-
- /**
- * The default number of threads to be used for rename operation.
- */
- public static final int DEFAULT_AZURE_RENAME_THREADS = 0;
-
- /**
- * The configuration property to set number of threads to be used for delete operation.
- */
- public static final String AZURE_DELETE_THREADS = "fs.azure.delete.threads";
-
- /**
- * The default number of threads to be used for delete operation.
- */
- public static final int DEFAULT_AZURE_DELETE_THREADS = 0;
-
- /**
- * The number of threads to be used for delete operation after reading user configuration.
- */
- private int deleteThreadCount = 0;
-
- /**
- * The number of threads to be used for rename operation after reading user configuration.
- */
- private int renameThreadCount = 0;
-
- private class NativeAzureFsInputStream extends FSInputStream {
- private InputStream in;
- private final String key;
- private long pos = 0;
- private boolean closed = false;
- private boolean isPageBlob;
-
- // File length, valid only for streams over block blobs.
- private long fileLength;
-
- NativeAzureFsInputStream(InputStream in, String key, long fileLength) {
- this.in = in;
- this.key = key;
- this.isPageBlob = store.isPageBlobKey(key);
- this.fileLength = fileLength;
- }
-
- /**
- * Return the size of the remaining available bytes
- * if the size is less than or equal to {@link Integer#MAX_VALUE},
- * otherwise, return {@link Integer#MAX_VALUE}.
- *
- * This is to match the behavior of DFSInputStream.available(),
- * which some clients may rely on (HBase write-ahead log reading in
- * particular).
- */
- @Override
- public synchronized int available() throws IOException {
- if (isPageBlob) {
- return in.available();
- } else {
- if (closed) {
- throw new IOException("Stream closed");
- }
- final long remaining = this.fileLength - pos;
- return remaining <= Integer.MAX_VALUE ?
- (int) remaining : Integer.MAX_VALUE;
- }
- }
-
- /*
- * Reads the next byte of data from the input stream. The value byte is
- * returned as an integer in the range 0 to 255. If no byte is available
- * because the end of the stream has been reached, the value -1 is returned.
- * This method blocks until input data is available, the end of the stream
- * is detected, or an exception is thrown.
- *
- * @returns int An integer corresponding to the byte read.
- */
- @Override
- public synchronized int read() throws FileNotFoundException, IOException {
- try {
- int result = 0;
- result = in.read();
- if (result != -1) {
- pos++;
- if (statistics != null) {
- statistics.incrementBytesRead(1);
- }
- }
- // Return to the caller with the result.
- //
- return result;
- } catch(EOFException e) {
- return -1;
- } catch(IOException e) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
-
- LOG.error("Encountered Storage Exception for read on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e, ((StorageException) innerException).getErrorCode());
-
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
- }
-
- throw e;
- }
- }
-
- /*
- * Reads up to len bytes of data from the input stream into an array of
- * bytes. An attempt is made to read as many as len bytes, but a smaller
- * number may be read. The number of bytes actually read is returned as an
- * integer. This method blocks until input data is available, end of file is
- * detected, or an exception is thrown. If len is zero, then no bytes are
- * read and 0 is returned; otherwise, there is an attempt to read at least
- * one byte. If no byte is available because the stream is at end of file,
- * the value -1 is returned; otherwise, at least one byte is read and stored
- * into b.
- *
- * @param b -- the buffer into which data is read
- *
- * @param off -- the start offset in the array b at which data is written
- *
- * @param len -- the maximum number of bytes read
- *
- * @ returns int The total number of byes read into the buffer, or -1 if
- * there is no more data because the end of stream is reached.
- */
- @Override
- public synchronized int read(byte[] b, int off, int len) throws FileNotFoundException, IOException {
- try {
- int result = 0;
- result = in.read(b, off, len);
- if (result > 0) {
- pos += result;
- }
-
- if (null != statistics && result > 0) {
- statistics.incrementBytesRead(result);
- }
-
- // Return to the caller with the result.
- return result;
- } catch(IOException e) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
-
- LOG.error("Encountered Storage Exception for read on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e, ((StorageException) innerException).getErrorCode());
-
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
- }
-
- throw e;
- }
- }
-
- @Override
- public int read(long position, byte[] buffer, int offset, int length)
- throws IOException {
- // SpotBugs reports bug type IS2_INCONSISTENT_SYNC here.
- // This report is not valid here.
- // 'this.in' is instance of BlockBlobInputStream and read(long, byte[], int, int)
- // calls it's Super class method when 'fs.azure.block.blob.buffered.pread.disable'
- // is configured false. Super class FSInputStream's implementation is having
- // proper synchronization.
- // When 'fs.azure.block.blob.buffered.pread.disable' is true, we want a lock free
- // implementation of blob read. Here we don't use any of the InputStream's
- // shared resource (buffer) and also don't change any cursor position etc.
- // So its safe to go with unsynchronized way of read.
- if (in instanceof PositionedReadable) {
- try {
- int result = ((PositionedReadable) this.in).read(position, buffer,
- offset, length);
- if (null != statistics && result > 0) {
- statistics.incrementBytesRead(result);
- }
- return result;
- } catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
- if (innerException instanceof StorageException) {
- LOG.error("Encountered Storage Exception for read on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e, ((StorageException) innerException).getErrorCode());
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
- }
- throw e;
- }
- }
- return super.read(position, buffer, offset, length);
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (!closed) {
- closed = true;
- IOUtils.closeStream(in);
- in = null;
- }
- }
-
- @Override
- public synchronized void seek(long pos) throws FileNotFoundException, EOFException, IOException {
- try {
- checkNotClosed();
- if (pos < 0) {
- throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
- }
- if (this.pos > pos) {
- if (in instanceof Seekable) {
- ((Seekable) in).seek(pos);
- this.pos = pos;
- } else {
- IOUtils.closeStream(in);
- in = store.retrieve(key);
- this.pos = in.skip(pos);
- }
- } else {
- this.pos += in.skip(pos - this.pos);
- }
- LOG.debug("Seek to position {}. Bytes skipped {}", pos,
- this.pos);
- } catch(IOException e) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- }
-
- throw e;
- } catch(IndexOutOfBoundsException e) {
- throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
- }
- }
-
- @Override
- public synchronized long getPos() throws IOException {
- return pos;
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
-
- /*
- * Helper method to check if a stream is closed.
- */
- private void checkNotClosed() throws IOException {
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
- }
- }
-
- /**
- * Azure output stream; wraps an inner stream of different types.
- */
- public class NativeAzureFsOutputStream extends OutputStream
- implements Syncable, StreamCapabilities {
- private String key;
- private String keyEncoded;
- private OutputStream out;
-
- public NativeAzureFsOutputStream(OutputStream out, String aKey,
- String anEncodedKey) throws IOException {
- // Check input arguments. The output stream should be non-null and the
- // keys
- // should be valid strings.
- if (null == out) {
- throw new IllegalArgumentException(
- "Illegal argument: the output stream is null.");
- }
-
- if (null == aKey || 0 == aKey.length()) {
- throw new IllegalArgumentException(
- "Illegal argument the key string is null or empty");
- }
-
- if (null == anEncodedKey || 0 == anEncodedKey.length()) {
- throw new IllegalArgumentException(
- "Illegal argument the encoded key string is null or empty");
- }
-
- // Initialize the member variables with the incoming parameters.
- this.out = out;
-
- setKey(aKey);
- setEncodedKey(anEncodedKey);
- }
-
- /**
- * Get a reference to the wrapped output stream.
- *
- * @return the underlying output stream
- */
- @InterfaceAudience.LimitedPrivate({"HDFS"})
- public OutputStream getOutStream() {
- return out;
- }
-
- @Override // Syncable
- public void hflush() throws IOException {
- if (out instanceof Syncable) {
- ((Syncable) out).hflush();
- } else {
- flush();
- }
- }
-
- @Override // Syncable
- public void hsync() throws IOException {
- if (out instanceof Syncable) {
- ((Syncable) out).hsync();
- } else {
- flush();
- }
- }
-
- /**
- * Propagate probe of stream capabilities to nested stream
- * (if supported), else return false.
- * @param capability string to query the stream support for.
- * @return true if the nested stream supports the specific capability.
- */
- @Override // StreamCapability
- public boolean hasCapability(String capability) {
- return StoreImplementationUtils.hasCapability(out, capability);
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (out != null) {
- // Close the output stream and decode the key for the output stream
- // before returning to the caller.
- //
- out.close();
- try {
- restoreKey();
- } finally {
- out = null;
- }
- }
- }
-
- /**
- * Writes the specified byte to this output stream. The general contract for
- * write is that one byte is written to the output stream. The byte to be
- * written is the eight low-order bits of the argument b. The 24 high-order
- * bits of b are ignored.
- *
- * @param b
- * 32-bit integer of block of 4 bytes
- */
- @Override
- public void write(int b) throws IOException {
- checkOpen();
- try {
- out.write(b);
- } catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
- LOG.error("Encountered Storage Exception for write on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
- }
- throw e;
- }
- }
-
- /**
- * Writes b.length bytes from the specified byte array to this output
- * stream. The general contract for write(b) is that it should have exactly
- * the same effect as the call write(b, 0, b.length).
- *
- * @param b
- * Block of bytes to be written to the output stream.
- */
- @Override
- public void write(byte[] b) throws IOException {
- checkOpen();
- try {
- out.write(b);
- } catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
- LOG.error("Encountered Storage Exception for write on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
- }
- throw e;
- }
- }
-
- /**
- * Writes <code>len</code> from the specified byte array starting at offset
- * <code>off</code> to the output stream. The general contract for write(b,
- * off, len) is that some of the bytes in the array <code>b</code>
- * are written to the output stream in order; element <code>b[off]</code>
- * is the first byte written and <code>b[off+len-1]</code> is the last
- * byte written by this operation.
- *
- * @param b
- * Byte array to be written.
- * @param off
- * Write this offset in stream.
- * @param len
- * Number of bytes to be written.
- */
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- checkOpen();
- try {
- out.write(b, off, len);
- } catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
- LOG.error("Encountered Storage Exception for write on Blob : {}"
- + " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
- }
- throw e;
- }
- }
-
- /**
- * Get the blob name.
- *
- * @return String Blob name.
- */
- public String getKey() {
- return key;
- }
-
- /**
- * Set the blob name.
- *
- * @param key
- * Blob name.
- */
- public void setKey(String key) {
- this.key = key;
- }
-
- /**
- * Get the blob name.
- *
- * @return String Blob name.
- */
- public String getEncodedKey() {
- return keyEncoded;
- }
-
- /**
- * Set the blob name.
- *
- * @param anEncodedKey
- * Blob name.
- */
- public void setEncodedKey(String anEncodedKey) {
- this.keyEncoded = anEncodedKey;
- }
-
- /**
- * Restore the original key name from the m_key member variable. Note: The
- * output file stream is created with an encoded blob store key to guarantee
- * load balancing on the front end of the Azure storage partition servers.
- * The create also includes the name of the original key value which is
- * stored in the m_key member variable. This method should only be called
- * when the stream is closed.
- */
- private void restoreKey() throws IOException {
- store.rename(getEncodedKey(), getKey());
- }
-
- /**
- * Check for the stream being open.
- * @throws IOException if the stream is closed.
- */
- private void checkOpen() throws IOException {
- if (out == null) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
- }
-
- }
-
- private URI uri;
- private NativeFileSystemStore store;
- private AzureNativeFileSystemStore actualStore;
- private Path workingDir;
- private AzureFileSystemInstrumentation instrumentation;
- private String metricsSourceName;
- private boolean isClosed = false;
- private static boolean suppressRetryPolicy = false;
- // A counter to create unique (within-process) names for my metrics sources.
- private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
- private boolean appendSupportEnabled = false;
- private boolean returnUriAsCanonicalServiceName = false;
- private DelegationTokenAuthenticatedURL authURL;
- private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
- private String credServiceUrl;
- private List<String> chownAllowedUsers;
- private List<String> chmodAllowedUsers;
- private List<String> daemonUsers;
- /**
- * Configuration key to enable authorization support in WASB.
- */
- public static final String KEY_AZURE_AUTHORIZATION =
- "fs.azure.authorization";
-
- /**
- * Default value for the authorization support in WASB.
- */
- private static final boolean DEFAULT_AZURE_AUTHORIZATION = false;
-
- /**
- * Flag controlling authorization support in WASB.
- */
- private boolean azureAuthorization = false;
-
- /**
- * Flag controlling Kerberos support in WASB.
- */
- private boolean kerberosSupportEnabled = false;
-
- /**
- * Authorizer to use when authorization support is enabled in
- * WASB.
- */
- private WasbAuthorizerInterface authorizer = null;
-
- private UserGroupInformation ugi;
-
- private WasbDelegationTokenManager wasbDelegationTokenManager;
-
- public NativeAzureFileSystem() {
- // set store in initialize()
- }
-
- public NativeAzureFileSystem(NativeFileSystemStore store) {
- this.store = store;
- }
-
- /**
- * Suppress the default retry policy for the Storage, useful in unit tests to
- * test negative cases without waiting forever.
- */
- @VisibleForTesting
- static void suppressRetryPolicy() {
- suppressRetryPolicy = true;
- }
-
- /**
- * Undo the effect of suppressRetryPolicy.
- */
- @VisibleForTesting
- static void resumeRetryPolicy() {
- suppressRetryPolicy = false;
- }
-
- /**
- * Creates a new metrics source name that's unique within this process.
- * @return metric source name
- */
- @VisibleForTesting
- public static String newMetricsSourceName() {
- int number = metricsSourceNameCounter.incrementAndGet();
- final String baseName = "AzureFileSystemMetrics";
- if (number == 1) { // No need for a suffix for the first one
- return baseName;
- } else {
- return baseName + number;
- }
- }
-
- /**
- * Checks if the given URI scheme is a scheme that's affiliated with the Azure
- * File System.
- *
- * @param scheme
- * The URI scheme.
- * @return true iff it's an Azure File System URI scheme.
- */
- private static boolean isWasbScheme(String scheme) {
- // The valid schemes are: asv (old name), asvs (old name over HTTPS),
- // wasb (new name), wasbs (new name over HTTPS).
- return scheme != null
- && (scheme.equalsIgnoreCase("asv") || scheme.equalsIgnoreCase("asvs")
- || scheme.equalsIgnoreCase("wasb") || scheme
- .equalsIgnoreCase("wasbs"));
- }
-
- /**
- * Puts in the authority of the default file system if it is a WASB file
- * system and the given URI's authority is null.
- *
- * @return The URI with reconstructed authority if necessary and possible.
- */
- private static URI reconstructAuthorityIfNeeded(URI uri, Configuration conf) {
- if (null == uri.getAuthority()) {
- // If WASB is the default file system, get the authority from there
- URI defaultUri = FileSystem.getDefaultUri(conf);
- if (defaultUri != null && isWasbScheme(defaultUri.getScheme())) {
- try {
- // Reconstruct the URI with the authority from the default URI.
- return new URI(uri.getScheme(), defaultUri.getAuthority(),
- uri.getPath(), uri.getQuery(), uri.getFragment());
- } catch (URISyntaxException e) {
- // This should never happen.
- throw new Error("Bad URI construction", e);
- }
- }
- }
- return uri;
- }
-
- @Override
- protected void checkPath(Path path) {
- // Make sure to reconstruct the path's authority if needed
- super.checkPath(new Path(reconstructAuthorityIfNeeded(path.toUri(),
- getConf())));
- }
-
- @Override
- public void initialize(URI uri, Configuration conf)
- throws IOException, IllegalArgumentException {
- // Check authority for the URI to guarantee that it is non-null.
- uri = reconstructAuthorityIfNeeded(uri, conf);
- if (null == uri.getAuthority()) {
- final String errMsg = String
- .format("Cannot initialize WASB file system, URI authority not recognized.");
- throw new IllegalArgumentException(errMsg);
- }
- super.initialize(uri, conf);
-
- if (store == null) {
- store = createDefaultStore(conf);
- }
-
- instrumentation = new AzureFileSystemInstrumentation(conf);
- if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
- // Make sure the metrics system is available before interacting with Azure
- AzureFileSystemMetricsSystem.fileSystemStarted();
- metricsSourceName = newMetricsSourceName();
- String sourceDesc = "Azure Storage Volume File System metrics";
- AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
- instrumentation);
- }
-
- store.initialize(uri, conf, instrumentation);
- setConf(conf);
- this.ugi = UserGroupInformation.getCurrentUser();
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser()
- .getShortUserName()).makeQualified(getUri(), getWorkingDirectory());
-
- this.appendSupportEnabled = conf.getBoolean(APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false);
- LOG.debug("NativeAzureFileSystem. Initializing.");
- LOG.debug(" blockSize = {}", store.getHadoopBlockSize());
-
- // Initialize thread counts from user configuration
- deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS);
- renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS);
-
- boolean useSecureMode = conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE,
- AzureNativeFileSystemStore.DEFAULT_USE_SECURE_MODE);
-
- this.azureAuthorization = useSecureMode &&
- conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION);
- this.kerberosSupportEnabled =
- conf.getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
-
- if (this.azureAuthorization) {
-
- this.authorizer =
- new RemoteWasbAuthorizerImpl();
- authorizer.init(conf);
-
- this.chmodAllowedUsers =
- Arrays.asList(conf.getTrimmedStrings(
- AZURE_CHMOD_USERLIST_PROPERTY_NAME,
- AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE));
- this.chownAllowedUsers =
- Arrays.asList(conf.getTrimmedStrings(
- AZURE_CHOWN_USERLIST_PROPERTY_NAME,
- AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE));
- this.daemonUsers =
- Arrays.asList(conf.getTrimmedStrings(
- AZURE_DAEMON_USERLIST_PROPERTY_NAME,
- AZURE_DAEMON_USERLIST_PROPERTY_DEFAULT_VALUE));
- }
-
- if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
- this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf);
- }
-
- this.returnUriAsCanonicalServiceName = conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
- }
-
- @Override
- public Path getHomeDirectory() {
- return makeQualified(new Path(
- USER_HOME_DIR_PREFIX_DEFAULT + "/" + this.ugi.getShortUserName()));
- }
-
- @VisibleForTesting
- public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) {
- this.authorizer = authorizer;
- }
-
- private NativeFileSystemStore createDefaultStore(Configuration conf) {
- actualStore = new AzureNativeFileSystemStore();
-
- if (suppressRetryPolicy) {
- actualStore.suppressRetryPolicy();
- }
- return actualStore;
- }
-
- /**
- * Azure Storage doesn't allow the blob names to end in a period,
- * so encode this here to work around that limitation.
- */
- private static String encodeTrailingPeriod(String toEncode) {
- Matcher matcher = TRAILING_PERIOD_PATTERN.matcher(toEncode);
- return matcher.replaceAll(TRAILING_PERIOD_PLACEHOLDER);
- }
-
- /**
- * Reverse the encoding done by encodeTrailingPeriod().
- */
- private static String decodeTrailingPeriod(String toDecode) {
- Matcher matcher = TRAILING_PERIOD_PLACEHOLDER_PATTERN.matcher(toDecode);
- return matcher.replaceAll(".");
- }
-
- /**
- * Convert the path to a key. By convention, any leading or trailing slash is
- * removed, except for the special case of a single slash.
- * @param path path converted to a key
- * @return key string
- */
- @VisibleForTesting
- public String pathToKey(Path path) {
- // Convert the path to a URI to parse the scheme, the authority, and the
- // path from the path object.
- URI tmpUri = path.toUri();
- String pathUri = tmpUri.getPath();
-
- // The scheme and authority is valid. If the path does not exist add a "/"
- // separator to list the root of the container.
- Path newPath = path;
- if ("".equals(pathUri)) {
- newPath = new Path(tmpUri.toString() + Path.SEPARATOR);
- }
-
- // Verify path is absolute if the path refers to a windows drive scheme.
- if (!newPath.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
-
- String key = null;
- key = newPath.toUri().getPath();
- key = removeTrailingSlash(key);
- key = encodeTrailingPeriod(key);
- if (key.length() == 1) {
- return key;
- } else {
- return key.substring(1); // remove initial slash
- }
- }
-
- // Remove any trailing slash except for the case of a single slash.
- private static String removeTrailingSlash(String key) {
- if (key.length() == 0 || key.length() == 1) {
- return key;
- }
- if (key.charAt(key.length() - 1) == '/') {
- return key.substring(0, key.length() - 1);
- } else {
- return key;
- }
- }
-
- static Path keyToPath(String key) {
- if (key.equals("/")) {
- return new Path("/"); // container
- }
- return new Path("/" + decodeTrailingPeriod(key));
- }
-
- /**
- * Get the absolute version of the path (fully qualified).
- * This is public for testing purposes.
- *
- * @param path path to be absolute path.
- * @return fully qualified path
- */
- @VisibleForTesting
- public Path makeAbsolute(Path path) {
- if (path.isAbsolute()) {
- return path;
- }
- return new Path(workingDir, path);
- }
-
- /**
- * For unit test purposes, retrieves the AzureNativeFileSystemStore store
- * backing this file system.
- *
- * @return The store object.
- */
- @VisibleForTesting
- public AzureNativeFileSystemStore getStore() {
- return actualStore;
- }
-
- NativeFileSystemStore getStoreInterface() {
- return store;
- }
-
- /**
- * @param requestingAccessForPath - The path to the ancestor/parent/subtree/file that needs to be
- * checked before granting access to originalPath
- * @param accessType - The type of access READ/WRITE being requested
- * @param operation - A string describing the operation being performed ("delete", "create" etc.).
- * @param originalPath - The originalPath that was being accessed
- */
- private void performAuthCheck(Path requestingAccessForPath, WasbAuthorizationOperations accessType,
- String operation, Path originalPath) throws WasbAuthorizationException, IOException {
-
- if (azureAuthorization && this.authorizer != null) {
-
- requestingAccessForPath = requestingAccessForPath.makeQualified(getUri(), getWorkingDirectory());
- originalPath = originalPath.makeQualified(getUri(), getWorkingDirectory());
-
- String owner = getOwnerForPath(requestingAccessForPath);
-
- if (!this.authorizer.authorize(requestingAccessForPath.toString(), accessType.toString(), owner)) {
- throw new WasbAuthorizationException(operation
- + " operation for Path : " + originalPath.toString() + " not allowed");
- }
-
- }
- }
-
- /**
- * Gets the metrics source for this file system.
- * This is mainly here for unit testing purposes.
- *
- * @return the metrics source.
- */
- public AzureFileSystemInstrumentation getInstrumentation() {
- return instrumentation;
- }
-
- /** This optional operation is not yet supported. */
- @Override
- public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
- throws IOException {
-
- if (!appendSupportEnabled) {
- throw new UnsupportedOperationException("Append Support not enabled");
- }
-
- LOG.debug("Opening file: {} for append", f);
-
- Path absolutePath = makeAbsolute(f);
-
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "append", absolutePath);
-
- String key = pathToKey(absolutePath);
- FileMetadata meta = null;
- try {
- meta = store.retrieveMetadata(key);
- } catch(Exception ex) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
-
- throw new FileNotFoundException(String.format("%s is not found", key));
- } else {
- throw ex;
- }
- }
-
- if (meta == null) {
- throw new FileNotFoundException(f.toString());
- }
-
- if (meta.isDirectory()) {
- throw new FileNotFoundException(f.toString()
- + " is a directory not a file.");
- }
-
- if (store.isPageBlobKey(key)) {
- throw new IOException("Append not supported for Page Blobs");
- }
-
- DataOutputStream appendStream = null;
-
- try {
- appendStream = store.retrieveAppendStream(key, bufferSize);
- } catch (Exception ex) {
-
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- throw new FileNotFoundException(String.format("%s is not found", key));
- } else {
- throw ex;
- }
- }
-
- return new FSDataOutputStream(appendStream, statistics);
- }
-
- @Override
- public FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
- return create(f, permission, overwrite, true,
- bufferSize, replication, blockSize, progress,
- (SelfRenewingLease) null);
- }
-
- /**
- * Get a self-renewing lease on the specified file.
- * @param path path whose lease to be renewed.
- * @return Lease
- * @throws AzureException when not being able to acquire a lease on the path
- */
- public SelfRenewingLease acquireLease(Path path) throws AzureException {
- String fullKey = pathToKey(makeAbsolute(path));
- return getStore().acquireLease(fullKey);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
-
- Path parent = f.getParent();
-
- // Get exclusive access to folder if this is a directory designated
- // for atomic rename. The primary use case of for HBase write-ahead
- // log file management.
- SelfRenewingLease lease = null;
- if (store.isAtomicRenameKey(pathToKey(f))) {
- try {
- lease = acquireLease(parent);
- } catch (AzureException e) {
-
- String errorCode = "";
- try {
- StorageException e2 = (StorageException) e.getCause();
- errorCode = e2.getErrorCode();
- } catch (Exception e3) {
- // do nothing if cast fails
- }
- if (errorCode.equals("BlobNotFound")) {
- throw new FileNotFoundException("Cannot create file " +
- f.getName() + " because parent folder does not exist.");
- }
-
- LOG.warn("Got unexpected exception trying to get lease on {} . {}",
- pathToKey(parent), e.getMessage());
- throw e;
- }
- }
-
- // See if the parent folder exists. If not, throw error.
- // The exists() check will push any pending rename operation forward,
- // if there is one, and return false.
- //
- // At this point, we have exclusive access to the source folder
- // via the lease, so we will not conflict with an active folder
- // rename operation.
- //
- // In the secure case, the call to exists will happen in the context
- // of the user that initiated the operation. In this case, we should
- // do the auth-check against ranger for the path.
- if (!exists(parent)) {
- try {
-
- // This'll let the keep-alive thread exit as soon as it wakes up.
- lease.free();
- } catch (Exception e) {
- LOG.warn("Unable to free lease because: {}", e.getMessage());
- }
- throw new FileNotFoundException("Cannot create file " +
- f.getName() + " because parent folder does not exist.");
- }
-
- // Create file inside folder.
- FSDataOutputStream out = null;
- try {
- out = create(f, permission, overwrite, false,
- bufferSize, replication, blockSize, progress, lease);
- } finally {
- // Release exclusive access to folder.
- try {
- if (lease != null) {
- lease.free();
- }
- } catch (Exception e) {
- NativeAzureFileSystemHelper.cleanup(LOG, out);
- String msg = "Unable to free lease on " + parent.toUri();
- LOG.error(msg);
- throw new IOException(msg, e);
- }
- }
- return out;
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
- EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
-
- // Check if file should be appended or overwritten. Assume that the file
- // is overwritten on if the CREATE and OVERWRITE create flags are set. Note
- // that any other combinations of create flags will result in an open new or
- // open with append.
- final EnumSet<CreateFlag> createflags =
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
- boolean overwrite = flags.containsAll(createflags);
-
- // Delegate the create non-recursive call.
- return this.createNonRecursive(f, permission, overwrite,
- bufferSize, replication, blockSize, progress);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path f,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
- return this.createNonRecursive(f, FsPermission.getFileDefault(),
- overwrite, bufferSize, replication, blockSize, progress);
- }
-
-
- /**
- * Create an Azure blob and return an output stream to use
- * to write data to it.
- *
- * @param f
- * @param permission
- * @param overwrite
- * @param createParent
- * @param bufferSize
- * @param replication
- * @param blockSize
- * @param progress
- * @param parentFolderLease Lease on parent folder (or null if
- * no lease).
- * @return
- * @throws IOException
- */
- private FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, boolean createParent, int bufferSize,
- short replication, long blockSize, Progressable progress,
- SelfRenewingLease parentFolderLease)
- throws FileAlreadyExistsException, IOException {
-
- LOG.debug("Creating file: {}", f.toString());
-
- if (containsColon(f)) {
- throw new IOException("Cannot create file " + f
- + " through WASB that has colons in the name");
- }
-
- Path absolutePath = makeAbsolute(f);
- Path ancestor = getAncestor(absolutePath);
-
- performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "create", absolutePath);
-
- return createInternal(f, permission, overwrite, parentFolderLease);
- }
-
-
- /**
- * This is the version of the create call that is meant for internal usage.
- * This version is not public facing and does not perform authorization checks.
- * It is used by the public facing create call and by FolderRenamePending to
- * create the internal -RenamePending.json file.
- * @param f the path to a file to be created.
- * @param permission for the newly created file.
- * @param overwrite specifies if the file should be overwritten.
- * @param parentFolderLease lease on the parent folder.
- * @return the output stream used to write data into the newly created file .
- * @throws IOException if an IO error occurs while attempting to delete the
- * path.
- *
- */
- protected FSDataOutputStream createInternal(Path f, FsPermission permission,
- boolean overwrite,
- SelfRenewingLease parentFolderLease)
- throws FileAlreadyExistsException, IOException {
-
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
-
- FileMetadata existingMetadata = store.retrieveMetadata(key);
- if (existingMetadata != null) {
- if (existingMetadata.isDirectory()) {
- throw new FileAlreadyExistsException("Cannot create file " + f
- + "; already exists as a directory.");
- }
- if (!overwrite) {
- throw new FileAlreadyExistsException("File already exists:" + f);
- }
- else {
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "create", absolutePath);
- }
- }
-
- Path parentFolder = absolutePath.getParent();
- if (parentFolder != null && parentFolder.getParent() != null) { // skip root
- // Update the parent folder last modified time if the parent folder
- // already exists.
- String parentKey = pathToKey(parentFolder);
- FileMetadata parentMetadata = store.retrieveMetadata(parentKey);
- if (parentMetadata != null && parentMetadata.isDirectory() &&
- parentMetadata.getBlobMaterialization() == BlobMaterialization.Explicit) {
- if (parentFolderLease != null) {
- store.updateFolderLastModifiedTime(parentKey, parentFolderLease);
- } else {
- updateParentFolderLastModifiedTime(key);
- }
- } else {
- // Make sure that the parent folder exists.
- // Create it using inherited permissions from the first existing directory going up the path
- Path firstExisting = parentFolder.getParent();
- FileMetadata metadata = store.retrieveMetadata(pathToKey(firstExisting));
- while(metadata == null) {
- // Guaranteed to terminate properly because we will eventually hit root, which will return non-null metadata
- firstExisting = firstExisting.getParent();
- metadata = store.retrieveMetadata(pathToKey(firstExisting));
- }
- mkdirs(parentFolder, metadata.getPermission(), true);
- }
- }
-
- // Mask the permission first (with the default permission mask as well).
- FsPermission masked = applyUMask(permission, UMaskApplyMode.NewFile);
- PermissionStatus permissionStatus = createPermissionStatus(masked);
-
- OutputStream bufOutStream;
- if (store.isPageBlobKey(key)) {
- // Store page blobs directly in-place without renames.
- bufOutStream = store.storefile(key, permissionStatus, key);
- } else {
- // This is a block blob, so open the output blob stream based on the
- // encoded key.
- //
- String keyEncoded = encodeKey(key);
-
-
- // First create a blob at the real key, pointing back to the temporary file
- // This accomplishes a few things:
- // 1. Makes sure we can create a file there.
- // 2. Makes it visible to other concurrent threads/processes/nodes what
- // we're
- // doing.
- // 3. Makes it easier to restore/cleanup data in the event of us crashing.
- store.storeEmptyLinkFile(key, keyEncoded, permissionStatus);
-
- // The key is encoded to point to a common container at the storage server.
- // This reduces the number of splits on the server side when load balancing.
- // Ingress to Azure storage can take advantage of earlier splits. We remove
- // the root path to the key and prefix a random GUID to the tail (or leaf
- // filename) of the key. Keys are thus broadly and randomly distributed over
- // a single container to ease load balancing on the storage server. When the
- // blob is committed it is renamed to its earlier key. Uncommitted blocks
- // are not cleaned up and we leave it to Azure storage to garbage collect
- // these
- // blocks.
- bufOutStream = new NativeAzureFsOutputStream(store.storefile(
- keyEncoded, permissionStatus, key), key, keyEncoded);
- }
- // Construct the data output stream from the buffered output stream.
- FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics);
-
-
- // Increment the counter
- instrumentation.fileCreated();
-
- // Return data output stream to caller.
- return fsOut;
- }
-
- @Override
- @Deprecated
- public boolean delete(Path path) throws IOException {
- return delete(path, true);
- }
-
- @Override
- public boolean delete(Path f, boolean recursive) throws IOException {
- return delete(f, recursive, false);
- }
-
- /**
- * Delete file or folder with authorization checks. Most of the code
- * is duplicate of the actual delete implementation and will be merged
- * once the performance and funcional aspects are guaranteed not to
- * regress existing delete semantics.
- */
- private boolean deleteWithAuthEnabled(Path f, boolean recursive,
- boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
-
- LOG.debug("Deleting file: {}", f);
-
- Path absolutePath = makeAbsolute(f);
- Path parentPath = absolutePath.getParent();
-
- // If delete is issued for 'root', parentPath will be null
- // In that case, we perform auth check for root itself before
- // proceeding for deleting contents under root.
- if (parentPath != null) {
- performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- } else {
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- }
-
- String key = pathToKey(absolutePath);
-
- // Capture the metadata for the path.
- FileMetadata metaFile = null;
- try {
- metaFile = store.retrieveMetadata(key);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
-
- return false;
- }
- throw e;
- }
-
- if (null == metaFile) {
- // The path to be deleted does not exist.
- return false;
- }
-
- FileMetadata parentMetadata = null;
- String parentKey = null;
- if (parentPath != null) {
- parentKey = pathToKey(parentPath);
-
- try {
- parentMetadata = store.retrieveMetadata(parentKey);
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
- if (innerException instanceof StorageException) {
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (isFileNotFoundException((StorageException) innerException)) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
- }
- throw e;
- }
-
- // Same case as unable to retrieve metadata
- if (parentMetadata == null) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
-
- if (!parentMetadata.isDirectory()) {
- // Invalid state: the parent path is actually a file. Throw.
- throw new AzureException("File " + f + " has a parent directory "
- + parentPath + " which is also a file. Can't resolve.");
- }
- }
-
- // The path exists, determine if it is a folder containing objects,
- // an empty folder, or a simple file and take the appropriate actions.
- if (!metaFile.isDirectory()) {
- // The path specifies a file. We need to check the parent path
- // to make sure it's a proper materialized directory before we
- // delete the file. Otherwise we may get into a situation where
- // the file we were deleting was the last one in an implicit directory
- // (e.g. the blob store only contains the blob a/b and there's no
- // corresponding directory blob a) and that would implicitly delete
- // the directory as well, which is not correct.
-
- if (parentPath != null && parentPath.getParent() != null) {// Not root
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the file {}. Creating the directory blob for"
- + " it in {}.", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- } else {
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- // check if the file can be deleted based on sticky bit check
- // This check will be performed only when authorization is enabled
- if (isStickyBitCheckViolated(metaFile, parentMetadata)) {
- throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
- + "File %s cannot be deleted.", parentPath, f));
- }
-
- try {
- if (store.delete(key)) {
- instrumentation.fileDeleted();
- } else {
- return false;
- }
- } catch(IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
-
- throw e;
- }
- } else {
- // The path specifies a folder. Recursively delete all entries under the
- // folder.
- LOG.debug("Directory Delete encountered: {}", f);
- if (parentPath != null && parentPath.getParent() != null) {
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the directory {}. Creating the directory blob for"
- + " it in {}. ", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- }
- }
-
- // check if the folder can be deleted based on sticky bit check on parent
- // This check will be performed only when authorization is enabled.
- if (!metaFile.getKey().equals("/")
- && isStickyBitCheckViolated(metaFile, parentMetadata)) {
-
- throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
- + "File %s cannot be deleted.", parentPath, f));
- }
-
- // Iterate through folder contents and get the list of files
- // and folders that can be deleted. We might encounter IOException
- // while listing blobs. In such cases, we return false.
- ArrayList<FileMetadata> fileMetadataList = new ArrayList<>();
- boolean isPartialDelete = false;
-
- // Start time for list operation
- long start = Time.monotonicNow();
-
- try {
- // Get list of files/folders that can be deleted
- // based on authorization checks and stickybit checks
- isPartialDelete = getFolderContentsToDelete(metaFile, fileMetadataList);
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
- throw e;
- }
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for delete operation: {} ms",
- fileMetadataList.size(), (end - start));
-
- // Here contents holds the list of metadata of the files and folders that can be deleted
- // under the path that is requested for delete(excluding itself).
- final FileMetadata[] contents = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
-
- if (contents.length > 0 && !recursive) {
- // The folder is non-empty and recursive delete was not specified.
- // Throw an exception indicating that a non-recursive delete was
- // specified for a non-empty folder.
- throw new IOException("Non-recursive delete of non-empty directory "
- + f);
- }
-
- // Delete all files / folders in current directory stored as list in 'contents'.
- AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
- @Override
- public boolean execute(FileMetadata file) throws IOException{
- if (!deleteFile(file.getKey(), file.isDirectory())) {
- LOG.warn("Attempt to delete non-existent {} {}",
- file.isDirectory() ? "directory" : "file",
- file.getKey());
- }
- return true;
- }
- };
-
- AzureFileSystemThreadPoolExecutor executor = getThreadPoolExecutor(this.deleteThreadCount,
- "AzureBlobDeleteThread", "Delete", key, AZURE_DELETE_THREADS);
-
- if (!executor.executeParallel(contents, task)) {
- LOG.error("Failed to delete files / subfolders in blob {}", key);
- return false;
- }
-
- if (metaFile.getKey().equals("/")) {
- LOG.error("Cannot delete root directory {}", f);
- return false;
- }
-
- // Delete the current directory if all underlying contents are deleted
- if (isPartialDelete || (store.retrieveMetadata(metaFile.getKey()) != null
- && !deleteFile(metaFile.getKey(), metaFile.isDirectory()))) {
- LOG.error("Failed delete directory : {}", f);
- return false;
- }
-
- // Update parent directory last modified time
- Path parent = absolutePath.getParent();
- if (parent != null && parent.getParent() != null) { // not root
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- // File or directory was successfully deleted.
- LOG.debug("Delete Successful for : {}", f);
- return true;
- }
-
- private boolean deleteWithoutAuth(Path f, boolean recursive,
- boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
-
- LOG.debug("Deleting file: {}", f);
-
- Path absolutePath = makeAbsolute(f);
- Path parentPath = absolutePath.getParent();
-
- String key = pathToKey(absolutePath);
-
- // Capture the metadata for the path.
- //
- FileMetadata metaFile = null;
- try {
- metaFile = store.retrieveMetadata(key);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
-
- return false;
- }
- throw e;
- }
-
- if (null == metaFile) {
- // The path to be deleted does not exist.
- return false;
- }
-
- // The path exists, determine if it is a folder containing objects,
- // an empty folder, or a simple file and take the appropriate actions.
- if (!metaFile.isDirectory()) {
- // The path specifies a file. We need to check the parent path
- // to make sure it's a proper materialized directory before we
- // delete the file. Otherwise we may get into a situation where
- // the file we were deleting was the last one in an implicit directory
- // (e.g. the blob store only contains the blob a/b and there's no
- // corresponding directory blob a) and that would implicitly delete
- // the directory as well, which is not correct.
-
- if (parentPath.getParent() != null) {// Not root
- String parentKey = pathToKey(parentPath);
-
- FileMetadata parentMetadata = null;
- try {
- parentMetadata = store.retrieveMetadata(parentKey);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (isFileNotFoundException((StorageException) innerException)) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
- }
- throw e;
- }
-
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (parentMetadata == null) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
-
- if (!parentMetadata.isDirectory()) {
- // Invalid state: the parent path is actually a file. Throw.
- throw new AzureException("File " + f + " has a parent directory "
- + parentPath + " which is also a file. Can't resolve.");
- }
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the file {}. Creating the directory blob for"
- + " it in {}.", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- } else {
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- try {
- if (store.delete(key)) {
- instrumentation.fileDeleted();
- } else {
- return false;
- }
- } catch(IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
-
- throw e;
- }
- } else {
- // The path specifies a folder. Recursively delete all entries under the
- // folder.
- LOG.debug("Directory Delete encountered: {}", f);
- if (parentPath.getParent() != null) {
- String parentKey = pathToKey(parentPath);
- FileMetadata parentMetadata = null;
-
- try {
- parentMetadata = store.retrieveMetadata(parentKey);
- } catch (IOException e) {
-
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException) {
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (isFileNotFoundException((StorageException) innerException)) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
- }
- throw e;
- }
-
- // Invalid State.
- // A FileNotFoundException is not thrown here as the API returns false
- // if the file not present. But not retrieving metadata here is an
- // unrecoverable state and can only happen if there is a race condition
- // hence throwing a IOException
- if (parentMetadata == null) {
- throw new IOException("File " + f + " has a parent directory "
- + parentPath + " whose metadata cannot be retrieved. Can't resolve");
- }
-
- if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
- LOG.debug("Found an implicit parent directory while trying to"
- + " delete the directory {}. Creating the directory blob for"
- + " it in {}. ", f, parentKey);
-
- store.storeEmptyFolder(parentKey,
- createPermissionStatus(FsPermission.getDefault()));
- }
- }
-
- // Start time for list operation
- long start = Time.monotonicNow();
- final FileMetadata[] contents;
-
- // List all the files in the folder with AZURE_UNBOUNDED_DEPTH depth.
- try {
- contents = store.list(key, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH);
- } catch (IOException e) {
- Throwable innerException = checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
-
- throw e;
- }
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for delete operation: {} ms", contents.length, (end - start));
-
- if (contents.length > 0) {
- if (!recursive) {
- // The folder is non-empty and recursive delete was not specified.
- // Throw an exception indicating that a non-recursive delete was
- // specified for a non-empty folder.
- throw new IOException("Non-recursive delete of non-empty directory "+ f);
- }
- }
-
- // Delete all files / folders in current directory stored as list in 'contents'.
- AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
- @Override
- public boolean execute(FileMetadata file) throws IOException{
- if (!deleteFile(file.getKey(), file.isDirectory())) {
- LOG.warn("Attempt to delete non-existent {} {}",
- file.isDirectory() ? "directory" : "file",
- file.getKey());
- }
- return true;
- }
- };
-
- AzureFileSystemThreadPoolExecutor executor = getThreadPoolExecutor(this.deleteThreadCount,
- "AzureBlobDeleteThread", "Delete", key, AZURE_DELETE_THREADS);
-
- if (!executor.executeParallel(contents, task)) {
- LOG.error("Failed to delete files / subfolders in blob {}", key);
- return false;
- }
-
- // Delete the current directory
- if (store.retrieveMetadata(metaFile.getKey()) != null
- && !deleteFile(metaFile.getKey(), metaFile.isDirectory())) {
- LOG.error("Failed delete directory : {}", f);
- return false;
- }
-
- // Update parent directory last modified time
- Path parent = absolutePath.getParent();
- if (parent != null && parent.getParent() != null) { // not root
- if (!skipParentFolderLastModifiedTimeUpdate) {
- updateParentFolderLastModifiedTime(key);
- }
- }
- }
-
- // File or directory was successfully deleted.
- LOG.debug("Delete Successful for : {}", f);
- return true;
- }
-
- /**
- * Delete the specified file or folder. The parameter
- * skipParentFolderLastModifiedTimeUpdate
- * is used in the case of atomic folder rename redo. In that case, there is
- * a lease on the parent folder, so (without reworking the code) modifying
- * the parent folder update time will fail because of a conflict with the
- * lease. Since we are going to delete the folder soon anyway so accurate
- * modified time is not necessary, it's easier to just skip
- * the modified time update.
- *
- * @param f file path to be deleted.
- * @param recursive specify deleting recursively or not.
- * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last
- * modified time.
- * @return true if and only if the file is deleted
- * @throws IOException Thrown when fail to delete file or directory.
- */
- public boolean delete(Path f, boolean recursive,
- boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
-
- if (this.azureAuthorization) {
- return deleteWithAuthEnabled(f, recursive,
- skipParentFolderLastModifiedTimeUpdate);
- } else {
- return deleteWithoutAuth(f, recursive,
- skipParentFolderLastModifiedTimeUpdate);
- }
- }
... 33379 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/02: HADOOP-19810. Replace FastByteComparisons with JDK Arrays.compareUnsigned (#8239) Contributed by Cheng Pan.
Posted by cn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 54d1dd00874f0a26c2a71d1e978fa08dbfe598e9
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Tue Feb 24 09:10:45 2026 +0800
HADOOP-19810. Replace FastByteComparisons with JDK Arrays.compareUnsigned (#8239) Contributed by Cheng Pan.
* HADOOP-19810. Replace FastByteComparisons with JDK Arrays.compareUnsigned
Reviewed-by: Akira Ajisaka <aa...@apache.org>
Signed-off-by: Shilun Fan <sl...@apache.org>
---
.../org/apache/hadoop/io/FastByteComparisons.java | 245 ---------------------
.../org/apache/hadoop/io/WritableComparator.java | 3 +-
2 files changed, 2 insertions(+), 246 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
deleted file mode 100644
index 1ef2119b688..00000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io;
-
-import java.lang.reflect.Field;
-import java.nio.ByteOrder;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import sun.misc.Unsafe;
-
-import org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes;
-
-/**
- * Utility code to do optimized byte-array comparison.
- * This is borrowed and slightly modified from Guava's {@link UnsignedBytes}
- * class to be able to compare arrays that start at non-zero offsets.
- */
-abstract class FastByteComparisons {
- static final Logger LOG = LoggerFactory.getLogger(FastByteComparisons.class);
-
- /**
- * Lexicographically compare two byte arrays.
- */
- public static int compareTo(byte[] b1, int s1, int l1, byte[] b2, int s2,
- int l2) {
- return LexicographicalComparerHolder.BEST_COMPARER.compareTo(
- b1, s1, l1, b2, s2, l2);
- }
-
-
- private interface Comparer<T> {
- abstract public int compareTo(T buffer1, int offset1, int length1,
- T buffer2, int offset2, int length2);
- }
-
- private static Comparer<byte[]> lexicographicalComparerJavaImpl() {
- return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
- }
-
-
- /**
- * Provides a lexicographical comparer implementation; either a Java
- * implementation or a faster implementation based on {@link Unsafe}.
- *
- * <p>Uses reflection to gracefully fall back to the Java implementation if
- * {@code Unsafe} isn't available.
- */
- private static class LexicographicalComparerHolder {
- static final String UNSAFE_COMPARER_NAME =
- LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
-
- static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
- /**
- * Returns the Unsafe-using Comparer, or falls back to the pure-Java
- * implementation if unable to do so.
- */
- static Comparer<byte[]> getBestComparer() {
- if (System.getProperty("os.arch").toLowerCase().startsWith("sparc")) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Lexicographical comparer selected for "
- + "byte aligned system architecture");
- }
- return lexicographicalComparerJavaImpl();
- }
- try {
- Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
-
- // yes, UnsafeComparer does implement Comparer<byte[]>
- @SuppressWarnings("unchecked")
- Comparer<byte[]> comparer =
- (Comparer<byte[]>) theClass.getEnumConstants()[0];
- if (LOG.isTraceEnabled()) {
- LOG.trace("Unsafe comparer selected for "
- + "byte unaligned system architecture");
- }
- return comparer;
- } catch (Throwable t) { // ensure we really catch *everything*
- if (LOG.isTraceEnabled()) {
- LOG.trace(t.getMessage());
- LOG.trace("Lexicographical comparer selected");
- }
- return lexicographicalComparerJavaImpl();
- }
- }
-
- private enum PureJavaComparer implements Comparer<byte[]> {
- INSTANCE;
-
- @Override
- public int compareTo(byte[] buffer1, int offset1, int length1,
- byte[] buffer2, int offset2, int length2) {
- // Short circuit equal case
- if (buffer1 == buffer2 &&
- offset1 == offset2 &&
- length1 == length2) {
- return 0;
- }
- // Bring WritableComparator code local
- int end1 = offset1 + length1;
- int end2 = offset2 + length2;
- for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
- int a = (buffer1[i] & 0xff);
- int b = (buffer2[j] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return length1 - length2;
- }
- }
-
- @SuppressWarnings("unused") // used via reflection
- private enum UnsafeComparer implements Comparer<byte[]> {
- INSTANCE;
-
- static final Unsafe theUnsafe;
-
- /** The offset to the first element in a byte array. */
- static final int BYTE_ARRAY_BASE_OFFSET;
-
- static {
- theUnsafe = (Unsafe) AccessController.doPrivileged(
- new PrivilegedAction<Object>() {
- @Override
- public Object run() {
- try {
- Field f = Unsafe.class.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- return f.get(null);
- } catch (NoSuchFieldException e) {
- // It doesn't matter what we throw;
- // it's swallowed in getBestComparer().
- throw new Error();
- } catch (IllegalAccessException e) {
- throw new Error();
- }
- }
- });
-
- BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
-
- // sanity check - this should never fail
- if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
- throw new AssertionError();
- }
- }
-
- static final boolean littleEndian =
- ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
-
- /**
- * Returns true if x1 is less than x2, when both values are treated as
- * unsigned.
- */
- static boolean lessThanUnsigned(long x1, long x2) {
- return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
- }
-
- /**
- * Lexicographically compare two arrays.
- *
- * @param buffer1 left operand
- * @param buffer2 right operand
- * @param offset1 Where to start comparing in the left buffer
- * @param offset2 Where to start comparing in the right buffer
- * @param length1 How much to compare from the left buffer
- * @param length2 How much to compare from the right buffer
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- @Override
- public int compareTo(byte[] buffer1, int offset1, int length1,
- byte[] buffer2, int offset2, int length2) {
- // Short circuit equal case
- if (buffer1 == buffer2 &&
- offset1 == offset2 &&
- length1 == length2) {
- return 0;
- }
- final int stride = 8;
- int minLength = Math.min(length1, length2);
- int strideLimit = minLength & ~(stride - 1);
- int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
- int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
- int i;
-
- /*
- * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
- * time is no slower than comparing 4 bytes at a time even on 32-bit.
- * On the other hand, it is substantially faster on 64-bit.
- */
- for (i = 0; i < strideLimit; i += stride) {
- long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
- long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
-
- if (lw != rw) {
- if (!littleEndian) {
- return lessThanUnsigned(lw, rw) ? -1 : 1;
- }
-
- /*
- * We want to compare only the first index where left[index] !=
- * right[index]. This corresponds to the least significant nonzero
- * byte in lw ^ rw, since lw and rw are little-endian.
- * Long.numberOfTrailingZeros(diff) tells us the least significant
- * nonzero bit, and zeroing out the first three bits of L.nTZ gives
- * us the shift to get that least significant nonzero byte. This
- * comparison logic is based on UnsignedBytes from Guava v21
- */
- int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7;
- return ((int) ((lw >>> n) & 0xFF)) - ((int) ((rw >>> n) & 0xFF));
- }
- }
-
- // The epilogue to cover the last (minLength % 8) elements.
- for (; i < minLength; i++) {
- int result = UnsignedBytes.compare(
- buffer1[offset1 + i],
- buffer2[offset2 + i]);
- if (result != 0) {
- return result;
- }
- }
- return length1 - length2;
- }
- }
- }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
index 05d4e3c5c53..e0a2edf1162 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
@@ -20,6 +20,7 @@
import java.io.DataInput;
import java.io.IOException;
+import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -229,7 +230,7 @@ public int compare(Object a, Object b) {
*/
public static int compareBytes(byte[] b1, int s1, int l1,
byte[] b2, int s2, int l2) {
- return FastByteComparisons.compareTo(b1, s1, l1, b2, s2, l2);
+ return Arrays.compareUnsigned(b1, s1, s1 + l1, b2, s2, s2 + l2);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org