You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/06/06 08:28:40 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HADOOP-13345
Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f02de2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f02de2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f02de2e
Branch: refs/heads/HADOOP-13345
Commit: 5f02de2e029fd739c8504740467b838d1d336e40
Parents: 2d06842 6aeda55
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Jun 5 13:10:05 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Jun 5 13:30:29 2017 -0700
----------------------------------------------------------------------
BUILDING.txt | 2 -
LICENSE.txt | 47 +-
dev-support/docker/Dockerfile | 2 +
hadoop-assemblies/pom.xml | 4 +-
.../resources/assemblies/hadoop-yarn-dist.xml | 5 +-
hadoop-build-tools/pom.xml | 12 +-
.../main/resources/checkstyle/checkstyle.xml | 4 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 4 +-
.../hadoop-client-check-invariants/pom.xml | 4 +-
.../hadoop-client-check-test-invariants/pom.xml | 4 +-
.../hadoop-client-integration-tests/pom.xml | 4 +-
.../hadoop-client-minicluster/pom.xml | 8 +-
.../hadoop-client-runtime/pom.xml | 5 +-
hadoop-client-modules/hadoop-client/pom.xml | 4 +-
hadoop-client-modules/pom.xml | 2 +-
.../hadoop-cloud-storage/pom.xml | 4 +-
hadoop-cloud-storage-project/pom.xml | 4 +-
.../hadoop-annotations/pom.xml | 4 +-
.../hadoop-auth-examples/pom.xml | 4 +-
hadoop-common-project/hadoop-auth/pom.xml | 19 +-
.../authentication/util/KerberosName.java | 5 +-
.../authentication/util/TestKerberosName.java | 4 +-
hadoop-common-project/hadoop-common/pom.xml | 9 +-
.../src/main/bin/hadoop-functions.sh | 24 -
.../src/main/conf/log4j.properties | 37 +-
.../org/apache/hadoop/conf/ConfigRedactor.java | 5 +-
.../org/apache/hadoop/conf/Configuration.java | 37 +-
.../hadoop/crypto/CryptoOutputStream.java | 4 +-
.../org/apache/hadoop/crypto/OpensslCipher.java | 4 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 2 +-
.../org/apache/hadoop/fs/BlockLocation.java | 21 +-
.../apache/hadoop/fs/CachingGetSpaceUsed.java | 3 +
.../fs/CommonConfigurationKeysPublic.java | 17 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 8 +-
.../src/main/java/org/apache/hadoop/fs/DU.java | 3 +-
.../apache/hadoop/fs/FSDataOutputStream.java | 10 +-
.../hadoop/fs/FSDataOutputStreamBuilder.java | 8 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 8 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 5 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/FilterFs.java | 4 +-
.../main/java/org/apache/hadoop/fs/Options.java | 2 +-
.../apache/hadoop/fs/RawLocalFileSystem.java | 2 +-
.../apache/hadoop/fs/StreamCapabilities.java | 67 +
.../hadoop/fs/http/AbstractHttpFileSystem.java | 153 +
.../apache/hadoop/fs/http/HttpFileSystem.java | 28 +
.../apache/hadoop/fs/http/HttpsFileSystem.java | 28 +
.../org/apache/hadoop/fs/http/package-info.java | 23 +
.../apache/hadoop/fs/sftp/SFTPFileSystem.java | 2 +-
.../hadoop/fs/shell/CommandWithDestination.java | 2 +-
.../org/apache/hadoop/fs/shell/FsUsage.java | 14 +-
.../apache/hadoop/fs/shell/XAttrCommands.java | 7 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 2 +-
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 2 +-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 8 +-
.../apache/hadoop/ha/ActiveStandbyElector.java | 4 +-
.../org/apache/hadoop/ha/HAServiceProtocol.java | 2 +-
.../java/org/apache/hadoop/ha/NodeFencer.java | 3 +-
.../org/apache/hadoop/ha/PowerShellFencer.java | 154 +
.../org/apache/hadoop/http/HttpServer2.java | 48 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 55 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 2 +-
.../apache/hadoop/io/compress/BZip2Codec.java | 20 +-
.../apache/hadoop/io/compress/CodecPool.java | 10 +-
.../io/compress/CompressionInputStream.java | 11 +-
.../io/compress/CompressionOutputStream.java | 16 +-
.../hadoop/io/compress/CompressorStream.java | 3 +-
.../hadoop/io/compress/DecompressorStream.java | 7 +-
.../compress/zlib/BuiltInGzipDecompressor.java | 2 +-
.../hadoop/io/compress/zlib/ZlibCompressor.java | 6 +-
.../io/compress/zlib/ZlibDecompressor.java | 2 +-
.../hadoop/io/erasurecode/CodecRegistry.java | 182 +
.../apache/hadoop/io/erasurecode/CodecUtil.java | 86 +-
.../apache/hadoop/io/erasurecode/ECSchema.java | 40 +-
.../io/erasurecode/ErasureCodeConstants.java | 1 +
.../rawcoder/DummyRawErasureCoderFactory.java | 12 +
.../NativeRSRawErasureCoderFactory.java | 13 +
.../NativeXORRawErasureCoderFactory.java | 13 +
.../rawcoder/RSLegacyRawDecoder.java | 270 +
.../rawcoder/RSLegacyRawEncoder.java | 129 +
.../RSLegacyRawErasureCoderFactory.java | 51 +
.../rawcoder/RSRawDecoderLegacy.java | 270 -
.../rawcoder/RSRawEncoderLegacy.java | 129 -
.../rawcoder/RSRawErasureCoderFactory.java | 13 +
.../RSRawErasureCoderFactoryLegacy.java | 38 -
.../rawcoder/RawErasureCoderFactory.java | 12 +
.../rawcoder/XORRawErasureCoderFactory.java | 13 +
.../hadoop/io/file/tfile/Compression.java | 2 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 2 +-
.../org/apache/hadoop/ipc/CallQueueManager.java | 1 +
.../apache/hadoop/ipc/DecayRpcScheduler.java | 33 +-
.../apache/hadoop/ipc/DefaultRpcScheduler.java | 4 +
.../org/apache/hadoop/ipc/RpcScheduler.java | 2 +
.../main/java/org/apache/hadoop/ipc/Server.java | 22 +-
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 4 +
.../apache/hadoop/metrics2/AbstractMetric.java | 16 +-
.../org/apache/hadoop/metrics2/MetricsTag.java | 13 +-
.../metrics2/impl/AbstractMetricsRecord.java | 16 +-
.../org/apache/hadoop/metrics2/impl/MsInfo.java | 9 +-
.../hadoop/metrics2/lib/MetricsInfoImpl.java | 11 +-
.../hadoop/metrics2/lib/MetricsRegistry.java | 20 +-
.../apache/hadoop/metrics2/package-info.java | 2 +-
.../hadoop/metrics2/source/JvmMetricsInfo.java | 9 +-
.../hadoop/metrics2/util/MetricsCache.java | 18 +-
.../apache/hadoop/metrics2/util/Servers.java | 4 +-
.../org/apache/hadoop/net/NetworkTopology.java | 12 +-
.../JniBasedUnixGroupsNetgroupMapping.java | 2 +-
.../apache/hadoop/security/SaslRpcServer.java | 4 +-
.../ShellBasedUnixGroupsNetgroupMapping.java | 2 +-
.../hadoop/security/UserGroupInformation.java | 2 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 15 +-
.../security/ssl/SSLHostnameVerifier.java | 11 -
.../web/DelegationTokenAuthenticator.java | 2 +-
.../hadoop/service/ServiceStateException.java | 66 +-
.../launcher/AbstractLaunchableService.java | 78 +
.../HadoopUncaughtExceptionHandler.java | 129 +
.../service/launcher/InterruptEscalator.java | 216 +
.../hadoop/service/launcher/IrqHandler.java | 178 +
.../service/launcher/LaunchableService.java | 95 +
.../service/launcher/LauncherArguments.java | 59 +
.../service/launcher/LauncherExitCodes.java | 183 +
.../launcher/ServiceLaunchException.java | 81 +
.../service/launcher/ServiceLauncher.java | 1044 ++++
.../service/launcher/ServiceShutdownHook.java | 112 +
.../hadoop/service/launcher/package-info.java | 462 ++
.../apache/hadoop/util/ComparableVersion.java | 29 +-
.../org/apache/hadoop/util/DataChecksum.java | 14 +-
.../apache/hadoop/util/ExitCodeProvider.java | 35 +
.../java/org/apache/hadoop/util/ExitUtil.java | 257 +-
.../hadoop/util/GenericOptionsParser.java | 125 +-
.../org/apache/hadoop/util/HostsFileReader.java | 263 +-
.../org/apache/hadoop/util/StringInterner.java | 37 +-
.../org/apache/hadoop/util/StringUtils.java | 54 +-
.../org/apache/hadoop/util/SysInfoWindows.java | 2 +-
.../org/apache/hadoop/io/nativeio/NativeIO.c | 2 +-
.../services/org.apache.hadoop.fs.FileSystem | 2 +
....erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/resources/core-default.xml | 35 +-
.../src/site/markdown/DeprecatedProperties.md | 1 +
.../src/site/markdown/FileSystemShell.md | 6 +-
.../hadoop-common/src/site/markdown/Metrics.md | 24 +-
.../src/site/markdown/RackAwareness.md | 12 +-
.../3.0.0-alpha3/CHANGES.3.0.0-alpha3.md | 71 +
.../3.0.0-alpha3/RELEASENOTES.3.0.0-alpha3.md | 22 +
.../apache/hadoop/conf/TestConfigRedactor.java | 22 +-
.../apache/hadoop/conf/TestConfiguration.java | 23 +-
.../crypto/TestCryptoStreamsForLocalFS.java | 10 +-
.../hadoop/crypto/TestCryptoStreamsNormal.java | 16 +-
.../kms/TestLoadBalancingKMSClientProvider.java | 2 +-
.../apache/hadoop/fs/FileContextTestHelper.java | 2 +-
.../hadoop/fs/FileSystemContractBaseTest.java | 142 +-
.../apache/hadoop/fs/FileSystemTestHelper.java | 2 +-
.../java/org/apache/hadoop/fs/TestFsShell.java | 3 +-
.../org/apache/hadoop/fs/TestGlobPattern.java | 2 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 37 +-
.../apache/hadoop/fs/TestSymlinkLocalFS.java | 6 +-
.../hadoop/fs/TestSymlinkLocalFSFileSystem.java | 8 +-
.../fs/contract/AbstractContractMkdirTest.java | 31 +-
.../contract/rawlocal/RawlocalFSContract.java | 2 +-
.../hadoop/fs/http/TestHttpFileSystem.java | 67 +
.../hadoop/fs/sftp/TestSFTPFileSystem.java | 14 +
.../fs/viewfs/ViewFileSystemBaseTest.java | 6 +-
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 2 +-
.../apache/hadoop/http/TestSSLHttpServer.java | 21 +-
.../java/org/apache/hadoop/io/TestIOUtils.java | 2 +-
.../apache/hadoop/io/TestSortedMapWritable.java | 2 +-
.../apache/hadoop/io/compress/TestCodec.java | 103 +-
.../hadoop/io/compress/TestCodecPool.java | 12 +-
.../erasurecode/TestCodecRawCoderMapping.java | 20 +-
.../io/erasurecode/TestCodecRegistry.java | 173 +
.../hadoop/io/erasurecode/TestECSchema.java | 47 +-
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../rawcoder/RawErasureCoderBenchmark.java | 2 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 4 +-
.../rawcoder/TestNativeRSRawCoder.java | 4 +-
.../rawcoder/TestNativeXORRawCoder.java | 4 +-
.../rawcoder/TestRSLegacyRawCoder.java | 33 +
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 4 +-
.../rawcoder/TestRSRawCoderInteroperable1.java | 4 +-
.../rawcoder/TestRSRawCoderInteroperable2.java | 4 +-
.../rawcoder/TestRSRawCoderLegacy.java | 33 -
.../erasurecode/rawcoder/TestRawCoderBase.java | 16 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 4 +-
.../rawcoder/TestXORRawCoderInteroperable1.java | 4 +-
.../rawcoder/TestXORRawCoderInteroperable2.java | 4 +-
.../apache/hadoop/io/nativeio/TestNativeIO.java | 6 +-
.../io/retry/UnreliableImplementation.java | 2 +-
.../java/org/apache/hadoop/ipc/TestIPC.java | 10 +-
.../org/apache/hadoop/net/TestNetUtils.java | 10 +-
.../security/TestUserGroupInformation.java | 2 +-
.../hadoop/security/ssl/TestSSLFactory.java | 10 +-
.../apache/hadoop/service/BreakableService.java | 23 +-
.../hadoop/service/TestCompositeService.java | 66 +-
.../AbstractServiceLauncherTestBase.java | 317 ++
.../launcher/ExitTrackingServiceLauncher.java | 59 +
.../service/launcher/TestServiceConf.java | 146 +
.../launcher/TestServiceInterruptHandling.java | 118 +
.../service/launcher/TestServiceLauncher.java | 213 +
.../TestServiceLauncherCreationFailures.java | 83 +
.../TestServiceLauncherInnerMethods.java | 95 +
.../ExceptionInExecuteLaunchableService.java | 96 +
.../testservices/FailInConstructorService.java | 33 +
.../testservices/FailInInitService.java | 38 +
.../testservices/FailInStartService.java | 37 +
.../testservices/FailingStopInStartService.java | 47 +
.../testservices/FailureTestService.java | 55 +
.../InitInConstructorLaunchableService.java | 63 +
.../testservices/LaunchableRunningService.java | 111 +
.../testservices/NoArgsAllowedService.java | 64 +
.../testservices/NullBindLaunchableService.java | 46 +
.../launcher/testservices/RunningService.java | 84 +
.../StoppingInStartLaunchableService.java | 49 +
.../StringConstructorOnlyService.java | 39 +
.../apache/hadoop/test/GenericTestUtils.java | 34 +
.../org/apache/hadoop/util/TestClassUtil.java | 2 +-
.../apache/hadoop/util/TestHostsFileReader.java | 19 +-
.../util/bloom/BloomFilterCommonTester.java | 4 +-
.../hadoop/util/bloom/TestBloomFilters.java | 4 +-
.../src/test/resources/contract/rawlocal.xml | 32 +-
.../src/test/resources/testConf.xml | 2 +-
.../scripts/hadoop_finalize_catalina_opts.bats | 56 -
hadoop-common-project/hadoop-kms/pom.xml | 8 +-
.../hadoop/crypto/key/kms/server/KMS.java | 2 +-
.../crypto/key/kms/server/KMSWebServer.java | 7 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 4 +-
hadoop-common-project/hadoop-nfs/pom.xml | 4 +-
.../org/apache/hadoop/mount/MountInterface.java | 2 +-
.../apache/hadoop/nfs/nfs3/Nfs3Constant.java | 2 +-
.../hadoop/nfs/nfs3/request/SetAttr3.java | 2 +-
.../org/apache/hadoop/oncrpc/RpcMessage.java | 2 +-
.../org/apache/hadoop/portmap/TestPortmap.java | 4 +-
hadoop-common-project/pom.xml | 4 +-
hadoop-dist/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 37 +-
.../hadoop/hdfs/DFSClientFaultInjector.java | 4 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 43 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 105 +-
.../hadoop/hdfs/DistributedFileSystem.java | 98 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 10 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 12 +
.../hdfs/client/HdfsClientConfigKeys.java | 4 +
.../hdfs/protocol/AddECPolicyResponse.java | 66 +
.../hdfs/protocol/ClientDatanodeProtocol.java | 5 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 37 +-
.../hdfs/protocol/DatanodeVolumeInfo.java | 122 +
.../hadoop/hdfs/protocol/DirectoryListing.java | 4 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 62 +-
.../hadoop/hdfs/protocol/ExtendedBlock.java | 16 +-
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 22 +-
.../hdfs/protocol/IllegalECPolicyException.java | 34 +
.../datatransfer/DataTransferProtocol.java | 19 +-
.../hdfs/protocol/datatransfer/Sender.java | 29 +-
.../ClientDatanodeProtocolTranslatorPB.java | 29 +
.../ClientNamenodeProtocolTranslatorPB.java | 48 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 65 +-
.../token/block/BlockTokenIdentifier.java | 66 +-
.../hdfs/server/protocol/SlowDiskReports.java | 5 +-
.../apache/hadoop/hdfs/util/ECPolicyLoader.java | 325 ++
.../src/main/proto/ClientDatanodeProtocol.proto | 9 +
.../src/main/proto/ClientNamenodeProtocol.proto | 7 +
.../src/main/proto/datatransfer.proto | 4 +
.../src/main/proto/erasurecoding.proto | 23 +
.../src/main/proto/hdfs.proto | 21 +
.../hdfs/protocol/TestErasureCodingPolicy.java | 86 +
.../hadoop/hdfs/util/TestECPolicyLoader.java | 313 ++
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 27 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 4 +-
.../fs/http/server/HttpFSServerWebServer.java | 7 +-
.../org/apache/hadoop/lib/server/Server.java | 2 +-
.../hadoop/lib/server/ServerException.java | 2 +-
.../apache/hadoop/lib/lang/TestXException.java | 2 +-
.../org/apache/hadoop/lib/wsrs/TestParam.java | 2 +-
.../hadoop-hdfs-native-client/pom.xml | 4 +-
.../src/main/native/libhdfs-tests/expect.h | 18 +
.../libhdfs-tests/test_libhdfs_threaded.c | 16 +
.../src/main/native/libhdfs/exception.c | 75 +-
.../src/main/native/libhdfs/exception.h | 16 +-
.../src/main/native/libhdfs/hdfs.c | 90 +
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 61 +
.../src/main/native/libhdfs/jni_helper.c | 95 +-
.../src/main/native/libhdfs/jni_helper.h | 35 +
.../libhdfs/os/posix/thread_local_storage.c | 54 +-
.../native/libhdfs/os/thread_local_storage.h | 61 +-
.../libhdfs/os/windows/thread_local_storage.c | 50 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 2 +-
.../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 2 +-
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 2 +-
.../dev-support/findbugsExcludeFile.xml | 5 +
.../jdiff/Apache_Hadoop_HDFS_3.0.0-alpha3.xml | 326 ++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +-
.../src/main/conf/user_ec_policies.xml.template | 72 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +-
.../hadoop/hdfs/net/DFSNetworkTopology.java | 24 +-
.../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 137 +
.../hadoop/hdfs/protocol/LayoutVersion.java | 2 +-
.../hdfs/protocol/datatransfer/Receiver.java | 20 +-
...tDatanodeProtocolServerSideTranslatorPB.java | 28 +
...tNamenodeProtocolServerSideTranslatorPB.java | 54 +-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 2 +-
.../qjournal/server/GetJournalEditServlet.java | 16 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 25 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 21 +-
.../hdfs/qjournal/server/JournalNode.java | 25 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 154 +-
.../block/BlockPoolTokenSecretManager.java | 34 +-
.../token/block/BlockTokenSecretManager.java | 93 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 47 +-
.../hadoop/hdfs/server/balancer/KeyManager.java | 41 +-
.../hdfs/server/balancer/NameNodeConnector.java | 16 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 18 +-
.../server/blockmanagement/BlockManager.java | 248 +-
.../BlockPlacementPolicyDefault.java | 40 +-
.../BlockUnderConstructionFeature.java | 9 +-
.../blockmanagement/CorruptReplicasMap.java | 2 +-
.../blockmanagement/DatanodeDescriptor.java | 68 +-
.../server/blockmanagement/DatanodeManager.java | 36 +-
.../blockmanagement/DatanodeStorageInfo.java | 9 +-
.../blockmanagement/InvalidateBlocks.java | 17 +-
.../blockmanagement/LowRedundancyBlocks.java | 6 +-
.../hdfs/server/common/HdfsServerConstants.java | 7 +-
.../apache/hadoop/hdfs/server/common/Util.java | 12 +-
.../hdfs/server/datanode/BPOfferService.java | 3 +-
.../hdfs/server/datanode/BPServiceActor.java | 5 +-
.../hdfs/server/datanode/BlockReceiver.java | 25 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 7 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 80 +-
.../server/datanode/DataNodeFaultInjector.java | 6 +-
.../server/datanode/DataNodeLayoutVersion.java | 2 +-
.../hdfs/server/datanode/DataStorage.java | 22 +-
.../hdfs/server/datanode/DataXceiver.java | 113 +-
.../hdfs/server/datanode/FileIoProvider.java | 23 +-
.../hdfs/server/datanode/LocalReplica.java | 13 +
.../server/datanode/ProfilingFileIoEvents.java | 19 +-
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../erasurecode/StripedBlockReader.java | 4 +-
.../erasurecode/StripedBlockWriter.java | 11 +-
.../erasurecode/StripedReconstructionInfo.java | 16 +-
.../datanode/erasurecode/StripedWriter.java | 5 +-
.../AvailableSpaceVolumeChoosingPolicy.java | 20 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 12 +-
.../RoundRobinVolumeChoosingPolicy.java | 2 +-
.../fsdataset/VolumeChoosingPolicy.java | 5 +-
.../impl/FsDatasetAsyncDiskService.java | 14 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 137 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 19 +-
.../datanode/web/SimpleHttpProxyHandler.java | 4 +-
.../hdfs/server/namenode/AclTransformation.java | 7 +-
.../hdfs/server/namenode/BackupImage.java | 2 +-
.../hdfs/server/namenode/CacheManager.java | 2 +-
.../hadoop/hdfs/server/namenode/Content.java | 2 +-
.../ContentSummaryComputationContext.java | 94 +-
.../namenode/ErasureCodingPolicyManager.java | 143 +-
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 13 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 14 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 4 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 10 +-
.../server/namenode/FSDirErasureCodingOp.java | 39 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 4 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 8 +-
.../hdfs/server/namenode/FSDirSnapshotOp.java | 7 +-
.../hdfs/server/namenode/FSDirSymlinkOp.java | 4 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 10 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 31 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 11 +-
.../hdfs/server/namenode/FSDirectory.java | 43 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 8 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 52 +-
.../server/namenode/FSImageFormatPBINode.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 153 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 121 +-
.../server/namenode/FileJournalManager.java | 18 +-
.../hadoop/hdfs/server/namenode/INode.java | 1 -
.../hdfs/server/namenode/INodeDirectory.java | 19 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 8 +-
.../hdfs/server/namenode/INodeReference.java | 2 -
.../hdfs/server/namenode/INodeSymlink.java | 1 -
.../namenode/INodeWithAdditionalFields.java | 2 +-
.../hdfs/server/namenode/INodesInPath.java | 78 +-
.../hadoop/hdfs/server/namenode/JournalSet.java | 33 +-
.../hdfs/server/namenode/LeaseManager.java | 118 +-
.../namenode/NNStorageRetentionManager.java | 27 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 2 +-
.../server/namenode/NameNodeLayoutVersion.java | 2 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 18 +-
.../server/namenode/ha/StandbyCheckpointer.java | 7 +-
.../snapshot/DirectorySnapshottableFeature.java | 24 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 19 +-
.../hdfs/server/namenode/snapshot/Snapshot.java | 1 -
.../namenode/snapshot/SnapshotManager.java | 24 +-
.../protocol/ReceivedDeletedBlockInfo.java | 2 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 122 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 122 +-
.../OfflineEditsXmlLoader.java | 2 +-
.../offlineImageViewer/ImageLoaderCurrent.java | 10 +-
.../java/org/apache/hadoop/hdfs/util/Diff.java | 2 +-
.../src/main/resources/hdfs-default.xml | 51 +-
.../src/main/webapps/hdfs/dfshealth.html | 2 +-
.../src/main/webapps/hdfs/dfshealth.js | 4 +-
.../src/site/markdown/HDFSCommands.md | 6 +-
.../src/site/markdown/HDFSErasureCoding.md | 34 +-
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +-
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 2 +-
.../org/apache/hadoop/TestRefreshCallQueue.java | 88 +-
.../hadoop/fs/TestSymlinkHdfsFileContext.java | 2 +-
.../hadoop/fs/TestSymlinkHdfsFileSystem.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 14 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 12 +
.../hadoop/hdfs/TestBlockStoragePolicy.java | 143 +
.../TestClientProtocolForPipelineRecovery.java | 92 +
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 25 +
.../TestDFSRSDefault10x4StripedInputStream.java | 34 -
...TestDFSRSDefault10x4StripedOutputStream.java | 35 -
...fault10x4StripedOutputStreamWithFailure.java | 35 -
.../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
...DFSStripedInputStreamWithRandomECPolicy.java | 45 +
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 37 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
...tputStreamWithFailureWithRandomECPolicy.java | 45 +
...FSStripedOutputStreamWithRandomECPolicy.java | 45 +
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 2 +-
.../hdfs/TestDFSXORStripedInputStream.java | 32 -
.../hdfs/TestDFSXORStripedOutputStream.java | 34 -
...estDFSXORStripedOutputStreamWithFailure.java | 35 -
.../hadoop/hdfs/TestDataTransferProtocol.java | 3 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 129 +
.../apache/hadoop/hdfs/TestFileCorruption.java | 1 +
.../apache/hadoop/hdfs/TestFileCreation.java | 2 +-
.../org/apache/hadoop/hdfs/TestGetBlocks.java | 84 +-
.../hadoop/hdfs/TestHDFSFileSystemContract.java | 12 +-
.../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 33 +
.../hadoop/hdfs/TestMaintenanceState.java | 222 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
.../hdfs/TestWriteBlockGetsBlockLengthHint.java | 6 +-
.../apache/hadoop/hdfs/UpgradeUtilities.java | 4 +-
.../net/TestDFSNetworkTopologyPerformance.java | 524 ++
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 37 +-
.../hdfs/qjournal/TestJournalNodeSync.java | 1 +
.../qjournal/client/DirectExecutorService.java | 154 +
.../hdfs/qjournal/client/TestQJMWithFaults.java | 3 +-
.../client/TestQuorumJournalManager.java | 3 +-
.../security/token/block/TestBlockToken.java | 186 +-
.../hdfs/server/balancer/TestBalancer.java | 108 +-
.../server/balancer/TestBalancerRPCDelay.java | 32 +
.../hdfs/server/balancer/TestKeyManager.java | 87 +
.../blockmanagement/BlockManagerTestUtil.java | 21 +
.../blockmanagement/TestBlockManager.java | 73 +-
.../TestBlockUnderConstructionFeature.java | 8 +-
.../blockmanagement/TestSlowDiskTracker.java | 4 +-
.../server/datanode/BlockReportTestBase.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 24 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 6 +-
.../server/datanode/TestBlockReplacement.java | 2 +-
.../datanode/TestDataNodeHotSwapVolumes.java | 40 +-
.../server/datanode/TestDataNodeMXBean.java | 4 +-
.../server/datanode/TestDataNodeMetrics.java | 7 +-
.../datanode/TestDataNodeVolumeMetrics.java | 4 +-
.../TestDataXceiverLazyPersistHint.java | 4 +-
.../hdfs/server/datanode/TestDiskError.java | 5 +-
.../server/datanode/TestFsDatasetCache.java | 62 +-
.../server/datanode/TestSimulatedFSDataset.java | 8 +-
.../extdataset/ExternalDatasetImpl.java | 14 +-
.../TestAvailableSpaceVolumeChoosingPolicy.java | 76 +-
.../TestRoundRobinVolumeChoosingPolicy.java | 29 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 6 +-
.../fsdataset/impl/TestFsVolumeList.java | 2 +-
.../fsdataset/impl/TestSpaceReservation.java | 67 +-
.../fsdataset/impl/TestWriteToReplica.java | 33 +-
.../TestDataNodeOutlierDetectionViaMetrics.java | 8 +
.../hdfs/server/namenode/NameNodeAdapter.java | 31 +-
.../namenode/TestBlockUnderConstruction.java | 45 +
.../hdfs/server/namenode/TestCheckpoint.java | 4 +-
.../TestCommitBlockSynchronization.java | 2 +-
.../namenode/TestDecommissioningStatus.java | 113 +-
.../TestDefaultBlockPlacementPolicy.java | 46 +
.../server/namenode/TestEnabledECPolicies.java | 11 +-
.../server/namenode/TestFSNamesystemLock.java | 16 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 32 +
.../hdfs/server/namenode/TestLeaseManager.java | 286 +-
.../namenode/TestMetadataVersionOutput.java | 4 +-
.../namenode/TestNameNodeStatusMXBean.java | 5 +-
.../namenode/TestNamenodeStorageDirectives.java | 330 ++
.../namenode/TestReconstructStripedBlocks.java | 2 +
.../hdfs/server/namenode/TestStartup.java | 5 +-
.../namenode/ha/TestFailureToReadEdits.java | 2 +-
.../namenode/ha/TestStandbyCheckpoints.java | 60 +
.../namenode/metrics/TestNameNodeMetrics.java | 2 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 298 ++
.../snapshot/TestRenameWithSnapshots.java | 201 -
.../namenode/snapshot/TestSnapshotDeletion.java | 75 -
.../snapshot/TestSnapshotDiffReport.java | 188 +-
.../namenode/snapshot/TestSnapshotManager.java | 11 +-
.../shortcircuit/TestShortCircuitCache.java | 8 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 20 +
.../hdfs/web/TestWebHdfsFileSystemContract.java | 18 +-
.../hadoop/hdfs/web/TestWebHdfsTokens.java | 8 +-
.../hadoop/tools/TestHdfsConfigFields.java | 2 -
.../java/org/apache/hadoop/tools/TestTools.java | 5 +-
.../test/resources/testErasureCodingConf.xml | 97 +
.../src/test/resources/testHDFSConf.xml | 2 +-
.../src/test/resources/test_ec_policies.xml | 65 +
hadoop-hdfs-project/pom.xml | 4 +-
hadoop-mapreduce-project/.gitignore | 1 -
hadoop-mapreduce-project/conf/mapred-site.xml | 21 +
.../conf/mapred-site.xml.template | 21 -
.../hadoop-mapreduce-client-app/pom.xml | 4 +-
.../hadoop/mapred/LocalContainerLauncher.java | 46 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 16 +-
.../v2/app/launcher/ContainerLauncherImpl.java | 2 +-
.../v2/app/job/impl/TestShuffleProvider.java | 1 -
.../app/webapp/TestAMWebServicesAttempts.java | 1 -
.../hadoop-mapreduce-client-common/pom.xml | 4 +-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 8 +-
.../mapred/TestMRWithDistributedCache.java | 2 +-
.../hadoop-mapreduce-client-core/pom.xml | 4 +-
.../org/apache/hadoop/mapred/AuditLogger.java | 154 -
.../apache/hadoop/mapred/FileInputFormat.java | 2 +-
.../apache/hadoop/mapred/FileOutputFormat.java | 2 +-
.../java/org/apache/hadoop/mapred/JVMId.java | 2 +-
.../org/apache/hadoop/mapred/JobClient.java | 2 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 7 +-
.../apache/hadoop/mapred/JobContextImpl.java | 3 +-
.../org/apache/hadoop/mapred/JobInProgress.java | 2 +-
.../org/apache/hadoop/mapred/JobTracker.java | 2 +-
.../org/apache/hadoop/mapred/Operation.java | 14 +-
.../java/org/apache/hadoop/mapred/Task.java | 2 +-
.../hadoop/mapred/TaskAttemptContextImpl.java | 3 +-
.../java/org/apache/hadoop/mapred/TaskLog.java | 2 +-
.../org/apache/hadoop/mapred/TaskStatus.java | 4 +-
.../hadoop/mapred/pipes/BinaryProtocol.java | 2 +-
.../org/apache/hadoop/mapreduce/Cluster.java | 2 +-
.../java/org/apache/hadoop/mapreduce/Job.java | 4 +-
.../org/apache/hadoop/mapreduce/JobStatus.java | 2 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +
.../mapreduce/lib/input/FileInputFormat.java | 2 +-
.../mapreduce/lib/jobcontrol/ControlledJob.java | 2 +-
.../mapreduce/lib/jobcontrol/JobControl.java | 2 +-
.../mapreduce/lib/output/FileOutputFormat.java | 2 +-
.../hadoop/mapreduce/task/reduce/Fetcher.java | 2 +-
.../hadoop/mapreduce/task/reduce/MapHost.java | 2 +-
.../org/apache/hadoop/mapred/TestClock.java | 2 +-
.../apache/hadoop/mapred/TestClusterStatus.java | 4 +-
.../org/apache/hadoop/mapred/TestJobConf.java | 2 +-
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +-
.../hadoop-mapreduce-client-hs/pom.xml | 4 +-
.../hadoop/mapreduce/v2/hs/HSAuditLogger.java | 2 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 7 +-
.../mapreduce/v2/hs/JobHistoryServer.java | 3 -
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 +-
.../org/apache/hadoop/mapred/YARNRunner.java | 132 +-
.../org/apache/hadoop/RandomTextWriterJob.java | 2 +-
.../org/apache/hadoop/fs/JHLogAnalyzer.java | 2 +-
.../java/org/apache/hadoop/fs/TestDFSIO.java | 4 +-
.../apache/hadoop/fs/slive/OperationOutput.java | 2 +-
.../org/apache/hadoop/fs/slive/PathFinder.java | 2 +-
.../apache/hadoop/hdfs/NNBenchWithoutMR.java | 6 +-
.../hadoop/mapred/GenericMRLoadGenerator.java | 2 +-
.../java/org/apache/hadoop/mapred/MRBench.java | 2 +-
.../apache/hadoop/mapred/TestAuditLogger.java | 161 -
.../mapred/TestMRTimelineEventHandling.java | 2 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 167 +
.../hadoop/mapred/ThreadedMapBenchmark.java | 2 +-
.../mapreduce/GenericMRLoadGenerator.java | 2 +-
.../hadoop/mapreduce/GrowingSleepJob.java | 68 +
.../apache/hadoop/mapreduce/LargeSorter.java | 2 +-
.../hadoop/mapreduce/RandomTextWriter.java | 2 +-
.../apache/hadoop/mapreduce/RandomWriter.java | 2 +-
.../mapreduce/TimelineServicePerformance.java | 2 +-
.../mapreduce/security/TestMRCredentials.java | 3 +-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +-
.../apache/hadoop/test/MapredTestDriver.java | 3 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 +-
.../src/CMakeLists.txt | 2 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 +-
.../hadoop-mapreduce-client/pom.xml | 4 +-
.../hadoop-mapreduce-examples/pom.xml | 4 +-
.../hadoop/examples/RandomTextWriter.java | 2 +-
.../apache/hadoop/examples/RandomWriter.java | 2 +-
.../org/apache/hadoop/examples/WordMean.java | 4 +-
.../org/apache/hadoop/examples/WordMedian.java | 4 +-
.../hadoop/examples/WordStandardDeviation.java | 4 +-
.../org/apache/hadoop/examples/pi/Parser.java | 8 +-
.../hadoop/examples/terasort/TeraGen.java | 2 +-
hadoop-mapreduce-project/pom.xml | 4 +-
hadoop-maven-plugins/pom.xml | 2 +-
.../resource/ServicesResourceTransformer.java | 4 +-
hadoop-minicluster/pom.xml | 4 +-
hadoop-project-dist/pom.xml | 24 +-
hadoop-project/pom.xml | 36 +-
hadoop-tools/hadoop-aliyun/pom.xml | 10 +-
.../oss/TestAliyunOSSFileSystemContract.java | 142 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 9 +-
hadoop-tools/hadoop-archives/pom.xml | 4 +-
hadoop-tools/hadoop-aws/pom.xml | 22 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 29 +-
.../hadoop/fs/s3a/S3AEncryptionMethods.java | 13 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 93 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 120 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 111 +-
.../site/markdown/tools/hadoop-aws/testing.md | 4 +-
.../fs/s3a/AbstractTestS3AEncryption.java | 4 +
.../fs/s3a/ITestS3ACopyFromLocalFile.java | 154 +
.../ITestS3AEncryptionAlgorithmValidation.java | 4 +-
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 308 +-
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 51 +
.../fs/s3a/ITestS3AFileSystemContract.java | 21 +-
.../hadoop/fs/s3a/TestSSEConfiguration.java | 223 +
.../NativeS3FileSystemContractBaseTest.java | 26 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 12 +-
.../src/site/markdown/index.md | 4 +-
.../fs/adl/live/AdlStorageConfiguration.java | 8 +-
.../adl/live/TestAdlFileSystemContractLive.java | 25 +-
hadoop-tools/hadoop-azure/pom.xml | 12 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 195 +-
.../fs/azure/RemoteSASKeyGeneratorImpl.java | 26 +-
.../fs/azure/RemoteWasbAuthorizerImpl.java | 45 +-
.../fs/azure/SyncableDataOutputStream.java | 12 +-
.../fs/azure/WasbAuthorizationOperations.java | 2 -
.../fs/azure/WasbAuthorizerInterface.java | 3 +-
.../fs/azure/security/WasbTokenRenewer.java | 6 -
.../hadoop-azure/src/site/markdown/index.md | 90 +-
.../hadoop/fs/azure/AbstractWasbTestBase.java | 7 +-
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 73 +-
.../TestNativeAzureFileSystemAuthorization.java | 576 +-
...veAzureFileSystemAuthorizationWithOwner.java | 122 +
...stNativeAzureFileSystemContractEmulator.java | 20 +-
.../TestNativeAzureFileSystemContractLive.java | 32 +-
...TestNativeAzureFileSystemContractMocked.java | 11 +-
...tiveAzureFileSystemContractPageBlobLive.java | 25 +-
.../fs/azure/TestWasbRemoteCallHelper.java | 18 +-
.../TestAzureFileSystemInstrumentation.java | 3 +-
hadoop-tools/hadoop-datajoin/pom.xml | 4 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
.../apache/hadoop/tools/DistCpConstants.java | 10 +-
.../org/apache/hadoop/tools/DistCpContext.java | 4 +
.../apache/hadoop/tools/DistCpOptionSwitch.java | 8 +
.../org/apache/hadoop/tools/DistCpOptions.java | 23 +-
.../org/apache/hadoop/tools/OptionsParser.java | 12 +
.../tools/mapred/RetriableFileCopyCommand.java | 11 +-
.../src/site/markdown/DistCp.md.vm | 1 +
.../apache/hadoop/tools/TestDistCpOptions.java | 19 +-
hadoop-tools/hadoop-extras/pom.xml | 4 +-
hadoop-tools/hadoop-gridmix/pom.xml | 4 +-
.../hadoop/mapred/gridmix/InputStriper.java | 6 +-
.../TotalHeapUsageEmulatorPlugin.java | 16 +-
.../hadoop/mapred/gridmix/GridmixTestUtils.java | 31 +-
.../gridmix/TestGridmixMemoryEmulation.java | 2 +-
.../mapred/gridmix/TestGridmixSubmission.java | 3 +-
hadoop-tools/hadoop-kafka/pom.xml | 12 +-
.../hadoop/metrics2/impl/TestKafkaMetrics.java | 10 +-
hadoop-tools/hadoop-openstack/pom.xml | 19 +-
.../SwiftAuthenticationFailedException.java | 6 +-
.../exceptions/SwiftBadRequestException.java | 6 +-
.../SwiftInvalidResponseException.java | 9 +-
.../SwiftThrottledRequestException.java | 6 +-
.../apache/hadoop/fs/swift/http/CopyMethod.java | 41 -
.../hadoop/fs/swift/http/CopyRequest.java | 41 +
.../swift/http/HttpInputStreamWithRelease.java | 25 +-
.../hadoop/fs/swift/http/SwiftRestClient.java | 805 +--
.../snative/SwiftNativeFileSystemStore.java | 11 +-
.../hadoop/fs/swift/util/HttpResponseUtils.java | 121 +
.../fs/swift/TestSwiftFileSystemContract.java | 19 +-
.../TestSwiftFileSystemPartitionedUploads.java | 2 +-
.../fs/swift/http/TestSwiftRestClient.java | 2 +-
.../src/test/resources/log4j.properties | 3 -
hadoop-tools/hadoop-pipes/pom.xml | 4 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 +-
.../org/apache/hadoop/tools/rumen/TaskInfo.java | 29 +-
.../apache/hadoop/tools/rumen/ZombieJob.java | 9 +-
.../util/MapReduceJobPropertiesParser.java | 10 +-
hadoop-tools/hadoop-sls/pom.xml | 23 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 46 +-
.../src/main/data/2jobs2min-rumen-jh.json | 587 +--
.../hadoop/yarn/sls/ReservationClientUtil.java | 78 +
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 885 +++-
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 131 +-
.../yarn/sls/appmaster/MRAMSimulator.java | 9 +-
.../sls/resourcemanager/MockAMLauncher.java | 5 +
.../sls/scheduler/FairSchedulerMetrics.java | 20 +-
.../sls/scheduler/SLSCapacityScheduler.java | 33 +-
.../yarn/sls/scheduler/SLSFairScheduler.java | 31 +-
.../yarn/sls/scheduler/SchedulerWrapper.java | 4 +
.../hadoop/yarn/sls/scheduler/TaskRunner.java | 9 +-
.../hadoop/yarn/sls/synthetic/SynthJob.java | 306 ++
.../yarn/sls/synthetic/SynthJobClass.java | 180 +
.../sls/synthetic/SynthTraceJobProducer.java | 316 ++
.../hadoop/yarn/sls/synthetic/SynthUtils.java | 101 +
.../yarn/sls/synthetic/SynthWorkload.java | 121 +
.../hadoop/yarn/sls/synthetic/package-info.java | 22 +
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 52 +-
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 37 +-
.../src/site/markdown/SchedulerLoadSimulator.md | 150 +-
.../hadoop/yarn/sls/BaseSLSRunnerTest.java | 120 +
.../sls/TestReservationSystemInvariants.java | 73 +
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 90 +-
.../hadoop/yarn/sls/TestSynthJobGeneration.java | 96 +
.../yarn/sls/appmaster/TestAMSimulator.java | 2 +-
.../yarn/sls/scheduler/TestTaskRunner.java | 2 +-
.../hadoop/yarn/sls/utils/TestSLSUtils.java | 30 +
.../hadoop/yarn/sls/web/TestSLSWebApp.java | 28 +-
.../src/test/resources/capacity-scheduler.xml | 10 +
.../src/test/resources/fair-scheduler.xml | 8 +-
.../hadoop-sls/src/test/resources/inputsls.json | 55 +
.../hadoop-sls/src/test/resources/nodes.json | 84 +
.../src/test/resources/sls-runner.xml | 6 +-
.../hadoop-sls/src/test/resources/syn.json | 53 +
.../hadoop-sls/src/test/resources/yarn-site.xml | 10 +-
hadoop-tools/hadoop-streaming/pom.xml | 4 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 4 +-
hadoop-tools/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +-
.../yarn/api/ContainerManagementProtocol.java | 8 +-
.../yarn/api/records/QueueConfigurations.java | 150 +
.../hadoop/yarn/api/records/QueueInfo.java | 42 +
.../records/timelineservice/TimelineMetric.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../yarn/exceptions/ConfigurationException.java | 45 +
.../src/main/proto/yarn_protos.proto | 14 +
.../pom.xml | 4 +-
.../distributedshell/ApplicationMaster.java | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-applications/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +-
.../apache/hadoop/yarn/client/api/NMClient.java | 86 +-
.../yarn/client/api/async/NMClientAsync.java | 98 +-
.../api/async/impl/NMClientAsyncImpl.java | 264 +-
.../yarn/client/api/impl/NMClientImpl.java | 91 +
.../yarn/client/api/impl/YarnClientImpl.java | 10 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 58 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 6 +-
.../hadoop/yarn/client/TestRMFailover.java | 5 +-
.../api/async/impl/TestNMClientAsync.java | 359 +-
.../yarn/client/api/impl/TestAMRMClient.java | 2 +-
.../yarn/client/api/impl/TestNMClient.java | 141 +
.../hadoop/yarn/client/cli/TestLogsCLI.java | 2 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 4 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +-
.../pb/ApplicationSubmissionContextPBImpl.java | 3 +-
.../impl/pb/QueueConfigurationsPBImpl.java | 137 +
.../api/records/impl/pb/QueueInfoPBImpl.java | 98 +-
.../hadoop/yarn/client/ClientRMProxy.java | 4 +-
.../ConfiguredRMFailoverProxyProvider.java | 2 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 45 +-
.../RequestHedgingRMFailoverProxyProvider.java | 2 +-
.../apache/hadoop/yarn/client/ServerProxy.java | 5 +
.../yarn/client/api/impl/TimelineConnector.java | 9 +-
.../hadoop/yarn/event/AsyncDispatcher.java | 12 +-
.../apache/hadoop/yarn/event/Dispatcher.java | 10 -
.../hadoop/yarn/event/EventDispatcher.java | 17 +-
.../logaggregation/AggregatedLogFormat.java | 4 +
.../yarn/util/ProcfsBasedProcessTree.java | 37 +-
.../util/ResourceCalculatorProcessTree.java | 54 -
.../yarn/util/WindowsBasedProcessTree.java | 12 -
.../resource/DefaultResourceCalculator.java | 5 +
.../resource/DominantResourceCalculator.java | 5 +
.../yarn/util/resource/ResourceCalculator.java | 9 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../org/apache/hadoop/yarn/webapp/WebApp.java | 6 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 17 +-
.../src/main/resources/yarn-default.xml | 22 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 21 +-
.../hadoop/yarn/client/TestClientRMProxy.java | 109 +
.../client/api/impl/TestTimelineClient.java | 2 +
.../hadoop/yarn/event/DrainDispatcher.java | 11 +-
.../yarn/util/TestProcfsBasedProcessTree.java | 72 +-
.../yarn/util/TestWindowsBasedProcessTree.java | 12 -
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +-
.../pom.xml | 16 +-
...pplicationHistoryManagerOnTimelineStore.java | 37 +-
.../timeline/RollingLevelDBTimelineStore.java | 29 +-
.../TimelineDelegationTokenIdentifierData.java | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 32 +-
.../hadoop-yarn-server-common/pom.xml | 4 +-
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 4 +-
.../nodemanager/CMgrCompletedAppsEvent.java | 2 +-
.../CMgrCompletedContainersEvent.java | 2 +-
.../server/nodemanager/ContainerExecutor.java | 4 +-
.../nodemanager/DefaultContainerExecutor.java | 7 +-
.../server/nodemanager/DeletionService.java | 468 +-
.../nodemanager/LinuxContainerExecutor.java | 171 +-
.../yarn/server/nodemanager/NMAuditLogger.java | 2 +-
.../nodemanager/NodeHealthCheckerService.java | 54 +-
.../yarn/server/nodemanager/NodeManager.java | 5 -
.../server/nodemanager/NodeStatusUpdater.java | 6 +
.../nodemanager/NodeStatusUpdaterImpl.java | 398 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 63 +-
.../nodemanager/api/impl/pb/NMProtoUtils.java | 110 +
.../nodemanager/api/impl/pb/package-info.java | 25 +
.../containermanager/ContainerLocalization.java | 21 -
.../ContainerLocalizationImpl.java | 29 -
.../containermanager/ContainerManager.java | 2 -
.../containermanager/ContainerManagerImpl.java | 105 +-
.../application/ApplicationImpl.java | 27 +-
.../container/ContainerImpl.java | 9 +-
.../recovery/DeletionTaskRecoveryInfo.java | 73 +
.../deletion/recovery/package-info.java | 25 +
.../deletion/task/DeletionTask.java | 258 +
.../deletion/task/DeletionTaskType.java | 24 +
.../deletion/task/FileDeletionTask.java | 202 +
.../deletion/task/package-info.java | 25 +
.../launcher/ContainerLaunch.java | 270 +-
.../launcher/ContainerRelaunch.java | 9 +
.../linux/resources/CGroupsHandler.java | 6 +-
.../linux/resources/CGroupsHandlerImpl.java | 248 +-
.../DelegatingLinuxContainerRuntime.java | 9 +-
.../runtime/DockerLinuxContainerRuntime.java | 109 +-
.../JavaSandboxLinuxContainerRuntime.java | 77 +-
.../runtime/docker/DockerCommandExecutor.java | 191 +
.../linux/runtime/docker/package-info.java | 26 +
.../localizer/LocalResourcesTrackerImpl.java | 26 +-
.../localizer/ResourceLocalizationService.java | 50 +-
.../logaggregation/AppLogAggregatorImpl.java | 60 +-
.../loghandler/NonAggregatingLogHandler.java | 7 +-
.../monitor/ContainersMonitorImpl.java | 49 +-
.../util/CgroupsLCEResourcesHandler.java | 32 +-
.../impl/container-executor.c | 231 +-
.../impl/container-executor.h | 28 +-
.../container-executor/impl/get_executable.c | 2 +-
.../test/test-container-executor.c | 34 +
.../yarn_server_nodemanager_recovery.proto | 8 +
.../nodemanager/DummyContainerManager.java | 6 +-
.../TestDefaultContainerExecutor.java | 3 +-
.../server/nodemanager/TestDeletionService.java | 57 +-
.../nodemanager/TestLinuxContainerExecutor.java | 9 +-
.../TestLinuxContainerExecutorWithMocks.java | 217 +-
.../nodemanager/TestNodeHealthService.java | 11 +-
.../nodemanager/TestNodeManagerReboot.java | 99 +-
.../nodemanager/TestNodeManagerResync.java | 164 -
.../amrmproxy/BaseAMRMProxyTest.java | 21 +-
.../amrmproxy/TestAMRMProxyService.java | 65 +
.../TestAMRMProxyTokenSecretManager.java | 81 +
.../api/impl/pb/TestNMProtoUtils.java | 91 +
.../BaseContainerManagerTest.java | 20 +-
.../containermanager/TestContainerManager.java | 5 -
.../TestContainerManagerRecovery.java | 56 +-
.../containermanager/TestNMProxy.java | 30 +-
.../deletion/task/FileDeletionMatcher.java | 84 +
.../deletion/task/TestFileDeletionTask.java | 85 +
.../launcher/TestContainerLaunch.java | 65 +-
.../MockPrivilegedOperationCaptor.java | 68 +
.../linux/resources/TestCGroupsHandlerImpl.java | 407 +-
.../resources/TestResourceHandlerModule.java | 3 -
.../runtime/TestDockerContainerRuntime.java | 42 +-
.../TestJavaSandboxLinuxContainerRuntime.java | 126 +-
.../docker/TestDockerCommandExecutor.java | 218 +
.../runtime/docker/TestDockerLoadCommand.java | 48 +
.../runtime/docker/TestDockerRunCommand.java | 63 +
.../TestLocalResourcesTrackerImpl.java | 13 +-
.../TestResourceLocalizationService.java | 106 +-
.../TestAppLogAggregatorImpl.java | 15 +-
.../TestLogAggregationService.java | 17 +-
.../TestNonAggregatingLogHandler.java | 8 +-
.../TestContainersMonitorResourceChange.java | 3 +-
.../TestContainerSchedulerQueuing.java | 9 +-
.../TestNMLeveldbStateStoreService.java | 2 +-
.../util/TestCgroupsLCEResourcesHandler.java | 70 +-
...-container-executer-with-configuration-error | 20 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 5 +-
...ActiveStandbyElectorBasedElectorService.java | 5 +-
.../server/resourcemanager/AdminService.java | 6 +-
.../ApplicationMasterService.java | 2 +-
.../resourcemanager/NodesListManager.java | 30 +-
.../server/resourcemanager/RMAppManager.java | 21 +-
.../server/resourcemanager/RMAuditLogger.java | 2 +-
...MCriticalThreadUncaughtExceptionHandler.java | 19 +-
.../server/resourcemanager/RMFatalEvent.java | 67 +-
.../resourcemanager/RMFatalEventType.java | 1 +
.../server/resourcemanager/ResourceManager.java | 199 +-
.../metrics/AbstractSystemMetricsPublisher.java | 2 +-
.../monitor/SchedulingMonitor.java | 55 +-
.../CapacitySchedulerPreemptionContext.java | 5 +
.../CapacitySchedulerPreemptionUtils.java | 9 +-
.../FifoIntraQueuePreemptionPlugin.java | 329 +-
.../capacity/IntraQueueCandidatesSelector.java | 110 +-
.../IntraQueuePreemptionComputePlugin.java | 10 +-
.../ProportionalCapacityPreemptionPolicy.java | 42 +-
...QueuePriorityContainerCandidateSelector.java | 2 +-
.../monitor/capacity/TempAppPerPartition.java | 6 +-
.../monitor/capacity/TempQueuePerPartition.java | 14 +
.../monitor/capacity/TempUserPerPartition.java | 88 +
.../invariants/InvariantViolationException.java | 35 +
.../monitor/invariants/InvariantsChecker.java | 96 +
.../invariants/MetricsInvariantChecker.java | 195 +
.../ReservationInvariantsChecker.java | 63 +
.../monitor/invariants/package-info.java | 22 +
.../resourcemanager/recovery/RMStateStore.java | 12 +-
.../recovery/ZKRMStateStore.java | 506 +-
.../reservation/InMemoryPlan.java | 11 +
.../PeriodicRLESparseResourceAllocation.java | 167 +
.../resourcemanager/reservation/PlanView.java | 9 +
.../RLESparseResourceAllocation.java | 51 +
.../planning/AlignedPlannerWithGreedy.java | 15 +-
.../planning/GreedyReservationAgent.java | 13 +-
.../reservation/planning/IterativePlanner.java | 196 +-
.../reservation/planning/ReservationAgent.java | 23 +-
.../planning/SimpleCapacityReplanner.java | 8 +-
.../reservation/planning/StageAllocator.java | 10 +-
.../planning/StageAllocatorGreedy.java | 4 +-
.../planning/StageAllocatorGreedyRLE.java | 4 +-
.../planning/StageAllocatorLowCostAligned.java | 279 +-
.../planning/StageEarliestStart.java | 46 -
.../planning/StageEarliestStartByDemand.java | 106 -
.../StageEarliestStartByJobArrival.java | 39 -
.../planning/StageExecutionInterval.java | 47 +
.../StageExecutionIntervalByDemand.java | 144 +
.../StageExecutionIntervalUnconstrained.java | 73 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 24 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 6 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../resourcemanager/scheduler/NodeType.java | 12 +-
.../resourcemanager/scheduler/QueueMetrics.java | 21 +-
.../scheduler/capacity/AbstractCSQueue.java | 25 +
.../scheduler/capacity/CSQueueMetrics.java | 4 +-
.../CapacitySchedulerConfiguration.java | 8 +
.../capacity/CapacitySchedulerQueueManager.java | 6 +-
.../scheduler/capacity/LeafQueue.java | 11 +-
.../scheduler/capacity/UsersManager.java | 11 +-
.../allocator/AbstractContainerAllocator.java | 6 +-
.../fair/AllocationFileLoaderService.java | 3 +-
.../scheduler/fair/FSAppAttempt.java | 44 +-
.../scheduler/fair/FSLeafQueue.java | 9 +-
.../scheduler/fair/FSParentQueue.java | 4 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 22 +-
.../scheduler/fair/FSQueueMetrics.java | 4 +-
.../scheduler/fair/FSSchedulerNode.java | 8 +-
.../scheduler/fair/FairScheduler.java | 27 +-
.../scheduler/fair/FairSchedulerUtilities.java | 69 +
.../scheduler/fair/QueueManager.java | 6 +-
.../scheduler/fair/SchedulingPolicy.java | 38 +-
.../fair/policies/FairSharePolicy.java | 2 +-
.../resourcemanager/webapp/RMAppsBlock.java | 17 +-
.../resourcemanager/webapp/RMWebAppUtil.java | 149 +
.../yarn/server/resourcemanager/HATestUtil.java | 9 +-
.../yarn/server/resourcemanager/MockRM.java | 10 +-
.../server/resourcemanager/TestAppManager.java | 2 +
.../resourcemanager/TestClientRMService.java | 25 +
.../resourcemanager/TestRMDispatcher.java | 1 +
.../resourcemanager/TestRMStoreCommands.java | 15 +-
.../applicationsmanager/TestAMRestart.java | 140 +-
.../monitor/TestSchedulingMonitor.java | 19 +
...alCapacityPreemptionPolicyMockFramework.java | 89 +-
...ionalCapacityPreemptionPolicyIntraQueue.java | 30 +-
...cityPreemptionPolicyIntraQueueUserLimit.java | 899 ++++
...pacityPreemptionPolicyIntraQueueWithDRF.java | 178 +
.../invariants/TestMetricsInvariantChecker.java | 99 +
.../recovery/RMStateStoreTestBase.java | 10 +-
.../recovery/TestFSRMStateStore.java | 1 +
.../recovery/TestLeveldbRMStateStore.java | 1 +
.../recovery/TestMemoryRMStateStore.java | 2 +
.../recovery/TestZKRMStateStore.java | 573 +-
.../reservation/ReservationSystemTestUtil.java | 14 +
...TestPeriodicRLESparseResourceAllocation.java | 142 +
.../TestRLESparseResourceAllocation.java | 56 +-
.../planning/TestAlignedPlanner.java | 417 +-
.../planning/TestGreedyReservationAgent.java | 14 +-
.../planning/TestSimpleCapacityReplanner.java | 4 +-
.../scheduler/TestAbstractYarnScheduler.java | 2 +-
.../capacity/TestCapacityScheduler.java | 15 +-
.../scheduler/fair/TestFairScheduler.java | 15 +
.../fair/TestFairSchedulerPreemption.java | 4 +
.../fair/TestFairSchedulerUtilities.java | 67 +
.../fair/policies/TestEmptyQueues.java | 6 +-
.../src/test/resources/invariants.txt | 54 +
.../pom.xml | 4 +-
.../hadoop-yarn-server-tests/pom.xml | 4 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 22 +-
.../server/TestContainerManagerSecurity.java | 14 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 34 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 2 +-
.../storage/TimelineSchemaCreator.java | 144 +-
.../storage/application/ApplicationTable.java | 12 +-
.../storage/entity/EntityTable.java | 12 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 4 +-
.../collector/AppLevelTimelineCollector.java | 8 +-
.../PerNodeTimelineCollectorsAuxService.java | 7 +-
.../collector/TimelineCollector.java | 27 +-
.../collector/TimelineCollectorManager.java | 6 +-
.../reader/filter/TimelineFilterList.java | 2 +-
.../storage/FileSystemTimelineReaderImpl.java | 129 +-
.../collector/TestTimelineCollector.java | 95 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 4 +-
.../server/webproxy/amfilter/AmIpFilter.java | 6 +
.../server/webproxy/amfilter/TestAmFilter.java | 8 +
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +-
.../src/site/markdown/FairScheduler.md | 6 +
.../src/site/markdown/ResourceManagerHA.md | 4 +-
.../src/site/markdown/ResourceManagerRestart.md | 2 +-
.../src/site/markdown/TimelineServiceV2.md | 5 +-
.../hadoop-yarn/hadoop-yarn-ui/README.md | 64 +
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 156 +-
.../hadoop-yarn-ui/src/main/webapp/.bowerrc | 5 +-
.../src/main/webapp/WEB-INF/wro.xml | 9 +
.../webapp/app/adapters/yarn-app-attempt.js | 4 +-
.../webapp/app/adapters/yarn-app-flowrun.js | 33 +
.../webapp/app/adapters/yarn-app-timeline.js | 35 +
.../src/main/webapp/app/adapters/yarn-app.js | 2 +-
.../main/webapp/app/adapters/yarn-container.js | 18 +-
.../src/main/webapp/app/adapters/yarn-entity.js | 35 +
.../webapp/app/adapters/yarn-flow-activity.js | 30 +
.../webapp/app/adapters/yarn-flowrun-brief.js | 32 +
.../main/webapp/app/adapters/yarn-flowrun.js | 38 +
.../app/adapters/yarn-timeline-appattempt.js | 41 +
.../app/adapters/yarn-timeline-container.js | 40 +
.../webapp/app/components/em-table-html-cell.js | 23 +
.../main/webapp/app/components/nodes-heatmap.js | 18 +-
.../webapp/app/components/simple-bar-chart.js | 206 +
.../webapp/app/components/sunburst-chart.js | 261 +
.../main/webapp/app/components/timeline-view.js | 13 +-
.../webapp/app/controllers/app-table-columns.js | 213 +-
.../webapp/app/controllers/yarn-app-attempt.js | 32 +-
.../webapp/app/controllers/yarn-app-attempts.js | 40 -
.../src/main/webapp/app/controllers/yarn-app.js | 53 +-
.../webapp/app/controllers/yarn-app/attempts.js | 24 +
.../webapp/app/controllers/yarn-app/charts.js | 28 +
.../webapp/app/controllers/yarn-app/info.js | 32 +
.../webapp/app/controllers/yarn-apps/apps.js | 10 +-
.../app/controllers/yarn-apps/services.js | 25 -
.../app/controllers/yarn-flow-activity.js | 125 +
.../main/webapp/app/controllers/yarn-flow.js | 36 +
.../webapp/app/controllers/yarn-flow/info.js | 66 +
.../webapp/app/controllers/yarn-flow/runs.js | 178 +
.../main/webapp/app/controllers/yarn-flowrun.js | 50 +
.../webapp/app/controllers/yarn-flowrun/info.js | 157 +
.../app/controllers/yarn-flowrun/metrics.js | 126 +
.../webapp/app/controllers/yarn-nodes/table.js | 8 +
.../webapp/app/controllers/yarn-services.js | 89 +-
.../src/main/webapp/app/mixins/app-attempt.js | 66 +
.../main/webapp/app/models/yarn-app-attempt.js | 11 +-
.../main/webapp/app/models/yarn-app-flowrun.js | 52 +
.../main/webapp/app/models/yarn-app-timeline.js | 105 +
.../src/main/webapp/app/models/yarn-app.js | 6 +-
.../main/webapp/app/models/yarn-container.js | 9 +-
.../src/main/webapp/app/models/yarn-entity.js | 26 +
.../webapp/app/models/yarn-flow-activity.js | 28 +
.../webapp/app/models/yarn-flowrun-brief.js | 70 +
.../src/main/webapp/app/models/yarn-flowrun.js | 32 +
.../app/models/yarn-timeline-appattempt.js | 143 +
.../app/models/yarn-timeline-container.js | 63 +
.../src/main/webapp/app/router.js | 21 +-
.../main/webapp/app/routes/timeline-error.js | 29 +
.../main/webapp/app/routes/yarn-app-attempt.js | 32 +-
.../main/webapp/app/routes/yarn-app-attempts.js | 34 -
.../src/main/webapp/app/routes/yarn-app.js | 35 +-
.../main/webapp/app/routes/yarn-app/attempts.js | 37 +
.../main/webapp/app/routes/yarn-app/charts.js | 53 +
.../src/main/webapp/app/routes/yarn-app/info.js | 37 +
.../webapp/app/routes/yarn-apps/services.js | 33 -
.../webapp/app/routes/yarn-flow-activity.js | 37 +
.../src/main/webapp/app/routes/yarn-flow.js | 53 +
.../main/webapp/app/routes/yarn-flow/info.js | 22 +
.../main/webapp/app/routes/yarn-flow/runs.js | 22 +
.../webapp/app/routes/yarn-flowrun-metric.js | 107 +
.../src/main/webapp/app/routes/yarn-flowrun.js | 58 +
.../main/webapp/app/routes/yarn-flowrun/info.js | 22 +
.../webapp/app/routes/yarn-flowrun/metrics.js | 22 +
.../src/main/webapp/app/routes/yarn-services.js | 34 +
.../webapp/app/serializers/yarn-app-flowrun.js | 64 +
.../webapp/app/serializers/yarn-app-timeline.js | 76 +
.../src/main/webapp/app/serializers/yarn-app.js | 3 +-
.../main/webapp/app/serializers/yarn-entity.js | 51 +
.../app/serializers/yarn-flow-activity.js | 54 +
.../app/serializers/yarn-flowrun-brief.js | 64 +
.../main/webapp/app/serializers/yarn-flowrun.js | 63 +
.../main/webapp/app/serializers/yarn-queue.js | 22 +-
.../app/serializers/yarn-timeline-appattempt.js | 69 +
.../app/serializers/yarn-timeline-container.js | 71 +
.../src/main/webapp/app/styles/app.css | 92 +
.../main/webapp/app/templates/application.hbs | 19 +-
.../templates/components/app-attempt-table.hbs | 8 +-
.../app/templates/components/app-table.hbs | 6 +-
.../templates/components/container-table.hbs | 2 +-
.../templates/components/em-table-html-cell.hbs | 23 +
.../app/templates/components/timeline-view.hbs | 8 +-
.../webapp/app/templates/timeline-error.hbs | 22 +
.../webapp/app/templates/yarn-app-attempt.hbs | 46 +-
.../webapp/app/templates/yarn-app-attempts.hbs | 55 -
.../src/main/webapp/app/templates/yarn-app.hbs | 232 +-
.../webapp/app/templates/yarn-app/attempts.hbs | 29 +
.../webapp/app/templates/yarn-app/charts.hbs | 43 +
.../main/webapp/app/templates/yarn-app/info.hbs | 167 +
.../webapp/app/templates/yarn-app/loading.hbs | 23 +
.../src/main/webapp/app/templates/yarn-apps.hbs | 11 +-
.../webapp/app/templates/yarn-apps/apps.hbs | 2 +-
.../webapp/app/templates/yarn-apps/services.hbs | 25 -
.../webapp/app/templates/yarn-flow-activity.hbs | 45 +
.../src/main/webapp/app/templates/yarn-flow.hbs | 49 +
.../webapp/app/templates/yarn-flow/info.hbs | 53 +
.../webapp/app/templates/yarn-flow/runs.hbs | 131 +
.../app/templates/yarn-flowrun-metric.hbs | 34 +
.../main/webapp/app/templates/yarn-flowrun.hbs | 49 +
.../webapp/app/templates/yarn-flowrun/info.hbs | 128 +
.../app/templates/yarn-flowrun/metrics.hbs | 34 +
.../webapp/app/templates/yarn-nodes/table.hbs | 2 +-
.../main/webapp/app/templates/yarn-services.hbs | 85 +
.../src/main/webapp/app/utils/converter.js | 44 +-
.../src/main/webapp/app/utils/error-utils.js | 58 +
.../src/main/webapp/bower-shrinkwrap.json | 66 +
.../src/main/webapp/config/default-config.js | 3 +-
.../src/main/webapp/ember-cli-build.js | 16 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 11 +-
.../components/em-table-html-cell-test.js | 43 +
.../components/simple-bar-chart-test.js | 43 +
.../unit/adapters/yarn-app-timeline-test.js | 30 +
.../adapters/yarn-timeline-appattempt-test.js | 30 +
.../adapters/yarn-timeline-container-test.js | 30 +
.../unit/controllers/yarn-app-attempts-test.js | 30 -
.../unit/controllers/yarn-app/attempts-test.js | 30 +
.../unit/controllers/yarn-app/charts-test.js | 30 +
.../unit/controllers/yarn-app/info-test.js | 30 +
.../unit/controllers/yarn-apps/services-test.js | 30 -
.../unit/controllers/yarn-flow/info-test.js | 30 +
.../unit/controllers/yarn-flow/runs-test.js | 30 +
.../unit/controllers/yarn-flowrun/info-test.js | 30 +
.../controllers/yarn-flowrun/metrics-test.js | 30 +
.../tests/unit/mixins/app-attempt-test.js | 30 +
.../webapp/tests/unit/mixins/charts-test.js | 30 -
.../tests/unit/models/yarn-app-timeline-test.js | 30 +
.../models/yarn-timeline-appattempt-test.js | 30 +
.../unit/models/yarn-timeline-container-test.js | 30 +
.../tests/unit/routes/timeline-error-test.js | 29 +
.../tests/unit/routes/yarn-app-attempts-test.js | 29 -
.../tests/unit/routes/yarn-app/attempts-test.js | 29 +
.../tests/unit/routes/yarn-app/charts-test.js | 29 +
.../tests/unit/routes/yarn-app/info-test.js | 29 +
.../tests/unit/routes/yarn-flow/info-test.js | 29 +
.../tests/unit/routes/yarn-flow/runs-test.js | 29 +
.../tests/unit/routes/yarn-flowrun/info-test.js | 29 +
.../unit/routes/yarn-flowrun/metrics-test.js | 29 +
.../tests/unit/routes/yarn-services-test.js | 29 +
.../unit/serializers/yarn-app-timeline-test.js | 33 +
.../yarn-timeline-appattempt-test.js | 33 +
.../serializers/yarn-timeline-container-test.js | 33 +
.../hadoop-yarn-ui/src/main/webapp/yarn.lock | 4983 ++++++++++++++++++
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +-
hadoop-yarn-project/pom.xml | 4 +-
pom.xml | 2 +-
1148 files changed, 45884 insertions(+), 10731 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 17d2404,92e2135..5b2ba65
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@@ -705,26 -748,13 +748,27 @@@ public abstract class FileSystemContrac
/**
* This a sanity check to make sure that any filesystem's handling of
- * renames doesn't cause any regressions
+ * renames empty dirs doesn't cause any regressions.
+ */
+ public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable {
+ if (!renameSupported()) return;
+ Path parentdir = path("testRenameEmptyToDirWithSamePrefixAllowed");
+ fs.mkdirs(parentdir);
+ Path dest = path("testRenameEmptyToDirWithSamePrefixAllowedDest");
+ rename(parentdir, dest, true, false, true);
+ }
+
+ /**
+ * This a sanity check to make sure that any filesystem's handling of
+ * renames non-empty dirs doesn't cause any regressions.
*/
+ @Test
public void testRenameToDirWithSamePrefixAllowed() throws Throwable {
- if (!renameSupported()) return;
+ assumeTrue(renameSupported());
final Path parentdir = path("testRenameToDirWithSamePrefixAllowed");
fs.mkdirs(parentdir);
+ // Before renaming, we create one file under the source parent directory
+ createFile(new Path(parentdir, "mychild"));
final Path dest = path("testRenameToDirWithSamePrefixAllowedDest");
rename(parentdir, dest, true, false, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index e14e341,9e15b3f..0fb1197
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@@ -313,123 -320,4 +325,114 @@@ public final class Constants
@InterfaceAudience.Private
public static final int MAX_MULTIPART_COUNT = 10000;
+ /**
+ * Classname of the S3A-specific output committer factory. This
+ * is what must be declared when attempting to use
+ */
+ @InterfaceStability.Unstable
+ public static final String S3A_OUTPUT_COMMITTER_FACTORY =
+ "org.apache.hadoop.fs.s3a.commit.S3AOutputCommitterFactory";
+
+ /* Constants. */
+ public static final String S3_METADATA_STORE_IMPL =
+ "fs.s3a.metadatastore.impl";
+
+ /** Minimum period of time (in milliseconds) to keep metadata (may only be
+ * applied when a prune command is manually run).
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_CLI_PRUNE_AGE =
+ "fs.s3a.s3guard.cli.prune.age";
+
+ /**
+ * The region of the DynamoDB service.
+ *
+ * This config has no default value. If the user does not set this, the
+ * S3Guard will operate table in the associated S3 bucket region.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_REGION_KEY =
+ "fs.s3a.s3guard.ddb.region";
+
+ /**
+ * The DynamoDB table name to use.
+ *
+ * This config has no default value. If the user does not set this, the
+ * S3Guard implementation will use the respective S3 bucket name.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_NAME_KEY =
+ "fs.s3a.s3guard.ddb.table";
+
+ /**
+ * Whether to create the DynamoDB table if the table does not exist.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_CREATE_KEY =
+ "fs.s3a.s3guard.ddb.table.create";
+
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_CAPACITY_READ_KEY =
+ "fs.s3a.s3guard.ddb.table.capacity.read";
+ public static final long S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT = 500;
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY =
+ "fs.s3a.s3guard.ddb.table.capacity.write";
+ public static final long S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT = 100;
+
+ /**
+ * The maximum put or delete requests per BatchWriteItem request.
+ *
+ * Refer to Amazon API reference for this limit.
+ */
+ public static final int S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT = 25;
+
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_MAX_RETRIES =
+ "fs.s3a.s3guard.ddb.max.retries";
+ /**
+ * Max retries on batched DynamoDB operations before giving up and
+ * throwing an IOException. Default is {@value}. See core-default.xml for
+ * more detail.
+ */
+ public static final int S3GUARD_DDB_MAX_RETRIES_DEFAULT = 9;
+
+ /**
+ * Period of time (in milliseconds) to sleep between batches of writes.
+ * Currently only applies to prune operations, as they are naturally a
+ * lower priority than other operations.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY =
+ "fs.s3a.s3guard.ddb.background.sleep";
+ public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25;
+
+ /**
+ * V1 committer.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3A_OUTPUT_COMMITTER_MRV1 =
+ "org.apache.hadoop.fs.s3a.commit.S3OutputCommitterMRv1";
+
+ /**
+ * The default "Null" metadata store: {@value}.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_METASTORE_NULL
+ = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore";
+
+ /**
+ * Use Local memory for the metadata: {@value}.
+ * This is not coherent across processes and must be used for testing only.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_METASTORE_LOCAL
+ = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore";
+
+ /**
+ * Use DynamoDB for the metadata: {@value}.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_METASTORE_DYNAMO
+ = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
-
- @InterfaceAudience.Private
- public static final String SSE_C_NO_KEY_ERROR = S3AEncryptionMethods.SSE_C
- .getMethod() +" is enabled and no encryption key is provided.";
-
-
- @InterfaceAudience.Private
- public static final String SSE_S3_WITH_KEY_ERROR = S3AEncryptionMethods.SSE_S3
- .getMethod() +" is configured and an " + "encryption key is provided";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 7de94cc,872dd5f..4eb94ad
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@@ -1675,65 -1553,45 +1702,67 @@@ public class S3AFileSystem extends File
* @throws IOException other IO problems
* @throws AmazonClientException on failures inside the AWS SDK
*/
- private boolean innerMkdirs(Path f, FsPermission permission)
+ private boolean innerMkdirs(Path p, FsPermission permission)
throws IOException, FileAlreadyExistsException, AmazonClientException {
+ boolean createOnS3 = false;
+ Path f = qualify(p);
LOG.debug("Making directory: {}", f);
incrementStatistic(INVOCATION_MKDIRS);
- FileStatus fileStatus;
- try {
- fileStatus = getFileStatus(f);
+ List<Path> metadataStoreDirs = null;
+ if (hasMetadataStore()) {
+ metadataStoreDirs = new ArrayList<>();
+ }
- if (fileStatus.isDirectory()) {
- return true;
- } else {
- throw new FileAlreadyExistsException("Path is a file: " + f);
+ DirectoryStatus status = checkPathForDirectory(f);
+ if (status == DirectoryStatus.DOES_NOT_EXIST) {
+ createOnS3 = true;
+ if (metadataStoreDirs != null) {
+ metadataStoreDirs.add(f);
}
- } catch (FileNotFoundException e) {
- Path fPart = f.getParent();
- do {
- try {
- fileStatus = getFileStatus(fPart);
- if (fileStatus.isDirectory()) {
- break;
- }
- if (fileStatus.isFile()) {
- throw new FileAlreadyExistsException(String.format(
- "Can't make directory for path '%s' since it is a file.",
- fPart));
- }
- } catch (FileNotFoundException fnfe) {
- instrumentation.errorIgnored();
+ } else if (status == DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) {
+ if (metadataStoreDirs != null) {
+ metadataStoreDirs.add(f);
+ }
+ } else if (status == DirectoryStatus
+ .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) {
+ return true;
+ } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) {
+ throw new FileAlreadyExistsException("Path is a file: " + f);
+ }
+
+ // Walk path to root, ensuring closest ancestor is a directory, not file
+ Path fPart = f.getParent();
+ do {
+ status = checkPathForDirectory(fPart);
+ // The fake directory on S3 may not be visible immediately, but
+ // only the leaf node has a fake directory on S3 so we can treat both
+ // cases the same and just create a metadata store entry.
+ if (status == DirectoryStatus.DOES_NOT_EXIST || status ==
+ DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) {
+ if (metadataStoreDirs != null) {
+ metadataStoreDirs.add(fPart);
}
- fPart = fPart.getParent();
- } while (fPart != null);
+ } else if (status == DirectoryStatus
+ .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) {
+ // do nothing - just break out of the loop, make whatever child
+ // directories are needed on the metadata store, and return the
+ // result.
+ break;
+ } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) {
+ throw new FileAlreadyExistsException("Path is a file: " + f);
+ }
+ fPart = fPart.getParent();
+ } while (fPart != null);
+ if (createOnS3) {
String key = pathToKey(f);
createFakeDirectory(key);
- // this is complicated because getParent(a/b/c/) returns a/b/c, but
- // we want a/b. See HADOOP-14428 for more details.
- deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
- return true;
}
+ S3Guard.makeDirsOrdered(metadataStore, metadataStoreDirs, username);
- deleteUnnecessaryFakeDirectories(f.getParent());
++ // this is complicated because getParent(a/b/c/) returns a/b/c, but
++ // we want a/b. See HADOOP-14428 for more details.
++ deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
+ return true;
}
/**
@@@ -2025,16 -1756,33 +2051,34 @@@
// Since we have a local file, we don't need to stream into a temporary file
LocalFileSystem local = getLocal(getConf());
File srcfile = local.pathToFile(src);
+ if (!srcfile.exists()) {
+ throw new FileNotFoundException("No file: " + src);
+ }
+ if (!srcfile.isFile()) {
+ throw new FileNotFoundException("Not a file: " + src);
+ }
+ try {
+ FileStatus status = getFileStatus(dst);
+ if (!status.isFile()) {
+ throw new FileAlreadyExistsException(dst + " exists and is not a file");
+ }
+ if (!overwrite) {
+ throw new FileAlreadyExistsException(dst + " already exists");
+ }
+ } catch (FileNotFoundException e) {
+ // no destination, all is well
+ }
+ final String key = pathToKey(dst);
final ObjectMetadata om = newObjectMetadata(srcfile.length());
PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, srcfile);
- Upload up = putObject(putObjectRequest);
+ UploadInfo info = putObject(putObjectRequest);
+ Upload upload = info.getUpload();
ProgressableProgressListener listener = new ProgressableProgressListener(
- this, key, up, null);
- up.addProgressListener(listener);
+ this, key, upload, null);
+ upload.addProgressListener(listener);
try {
- up.waitForUploadResult();
+ upload.waitForUploadResult();
} catch (InterruptedException e) {
throw new InterruptedIOException("Interrupted copying " + src
+ " to " + dst + ", cancelling");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
index 0000000,71776ac..7dc286d
mode 000000,100644..100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
@@@ -1,0 -1,153 +1,154 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.fs.s3a;
+
+ import java.io.File;
+ import java.io.FileNotFoundException;
+ import java.io.IOException;
+ import java.nio.charset.Charset;
+
+ import org.junit.Test;
+
+ import org.apache.commons.io.Charsets;
+ import org.apache.commons.io.FileUtils;
+ import org.apache.commons.io.IOUtils;
+ import org.apache.hadoop.fs.FileAlreadyExistsException;
++import org.apache.hadoop.fs.FileStatus;
+ import org.apache.hadoop.fs.Path;
+
+ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+ /**
+ * Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}.
+ */
+ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
+ private static final Charset ASCII = Charsets.US_ASCII;
+
+ private File file;
+
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ if (file != null) {
+ file.delete();
+ }
+ }
+
+ @Test
+ public void testCopyEmptyFile() throws Throwable {
+ file = File.createTempFile("test", ".txt");
+ Path dest = upload(file, true);
+ assertPathExists("uploaded file", dest);
+ }
+
+ @Test
+ public void testCopyFile() throws Throwable {
+ String message = "hello";
+ file = createTempFile(message);
+ Path dest = upload(file, true);
+ assertPathExists("uploaded file not found", dest);
+ S3AFileSystem fs = getFileSystem();
- S3AFileStatus status = fs.getFileStatus(dest);
++ FileStatus status = fs.getFileStatus(dest);
+ assertEquals("File length of " + status,
+ message.getBytes(ASCII).length, status.getLen());
+ assertFileTextEquals(dest, message);
+ }
+
+ public void assertFileTextEquals(Path path, String expected)
+ throws IOException {
+ assertEquals("Wrong data in " + path,
+ expected, IOUtils.toString(getFileSystem().open(path), ASCII));
+ }
+
+ @Test
+ public void testCopyFileNoOverwrite() throws Throwable {
+ file = createTempFile("hello");
+ Path dest = upload(file, true);
+ intercept(FileAlreadyExistsException.class,
+ () -> upload(file, false));
+ }
+
+ @Test
+ public void testCopyFileOverwrite() throws Throwable {
+ file = createTempFile("hello");
+ Path dest = upload(file, true);
+ String updated = "updated";
+ FileUtils.write(file, updated, ASCII);
+ upload(file, true);
+ assertFileTextEquals(dest, updated);
+ }
+
+ @Test
+ public void testCopyFileNoOverwriteDirectory() throws Throwable {
+ file = createTempFile("hello");
+ Path dest = upload(file, true);
+ S3AFileSystem fs = getFileSystem();
+ fs.delete(dest, false);
+ fs.mkdirs(dest);
+ intercept(FileAlreadyExistsException.class,
+ () -> upload(file, true));
+ }
+
+ @Test
+ public void testCopyMissingFile() throws Throwable {
+ file = File.createTempFile("test", ".txt");
+ file.delete();
+ // first upload to create
+ intercept(FileNotFoundException.class, "No file",
+ () -> upload(file, true));
+ }
+
+ @Test
+ public void testCopyDirectoryFile() throws Throwable {
+ file = File.createTempFile("test", ".txt");
+ // first upload to create
+ intercept(FileNotFoundException.class, "Not a file",
+ () -> upload(file.getParentFile(), true));
+ }
+
+
+ @Test
+ public void testLocalFilesOnly() throws Throwable {
+ Path dst = path("testLocalFilesOnly");
+ intercept(IllegalArgumentException.class,
+ () -> {
+ getFileSystem().copyFromLocalFile(false, true, dst, dst);
+ return "copy successful";
+ });
+ }
+
+ public Path upload(File srcFile, boolean overwrite) throws IOException {
+ Path src = new Path(srcFile.toURI());
+ Path dst = path(srcFile.getName());
+ getFileSystem().copyFromLocalFile(false, overwrite, src, dst);
+ return dst;
+ }
+
+ /**
+ * Create a temp file with some text.
+ * @param text text for the file
+ * @return the file
+ * @throws IOException on a failure
+ */
+ public File createTempFile(String text) throws IOException {
+ File f = File.createTempFile("test", ".txt");
+ FileUtils.write(f, text, ASCII);
+ return f;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
index 6a0f878,1b49d07..2d47bb3
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@@ -26,9 -27,9 +27,11 @@@ import org.slf4j.LoggerFactory
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
+ import static org.junit.Assume.*;
+ import static org.junit.Assert.*;
+import java.io.IOException;
+
/**
* Tests a live S3 system. If your keys and bucket aren't specified, all tests
* are marked as passed.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org