You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2019/11/22 22:30:00 UTC
[hadoop-ozone] branch HDDS-1880-Decom updated (88147d0 -> 0c97903)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from 88147d0 Merge branch 'master' into HDDS-1880-Decom
add 8e00c2e HDDS-2395. Handle completeMPU scenarios to match with aws s3 behavior. (#109)
add f2f97ea HDDS-2399. Update mailing list information. (#126)
add 6a450fb HDDS-2427. Exclude webapps from hadoop-ozone-filesystem-lib-current uber jar
add 9ce2dd7 HDDS-1701. Move dockerbin script to libexec
add f928a0b HDDS-2273. Avoid buffer copying in GrpcReplicationService. Contributed by Attila Doroszlai
add 93d0db4 HDDS-2410. Ozoneperf docker cluster should use privileged containers (#124)
add f7ba616 HDDS-2417 Add the list trash command to the client side (#138)
add a6f80c0 HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE
add 3b0a49e HDDS-2415. Completely disable tracer if hdds.tracing.enabled=false
add ee8b89a HDDS-1868. Ozone pipelines should be marked as ready only after the leader election is complete. (#23)
add 1c1aecf HDDS-2456. Add explicit base image version for images derived from ozone-runner
add 1e71b63 HDDS-2462. Add jq dependency in Contribution guideline (#145)
add 2211d63 HDDS-2460. Default checksum type is wrong in description
add c199a29 HDDS-2451. Use lazy string evaluation in preconditions
add 4337bb0 HDDS-2403. Remove leftover reference to OUTPUT_FILE from shellcheck.sh
add ef2a9f4 HDDS-2445. Replace ToStringBuilder in BlockData
add d816b64 HDDS-2452: Wrong condition for re-scheduling in ReportPublisher (#151)
add 1f8c66a HDDS-2463. Reduce unnecessary getServiceInfo calls. Contributed by Xiaoyu Yao. (#146)
add fe85445 HDDS-2464. Avoid unnecessary allocations for FileChannel.open call. (#147)
add 42b6909 HDDS-2458. Avoid list copy in ChecksumData. (#141)
add 86c76ff HDDS-2400. Enable github actions based builds for Ozone (#122)
add cb66b48 HDDS-2412. Define description/topics/merge strategy for the github repository (#125)
add c83d5bc HDDS-2364. Add OM metrics to find the false positive rate for the keyMayExist. (#101)
add e39134d HDDS-1847: Datanode Kerberos principal and keytab config key looks inconsistent (#115)
add 659385e HDDS-2469. Avoid changing client-side key metadata (#154)
add 528cea3 HDDS-1940. Closing open container via scmcli gives false error message. (#153)
add e350aef HDDS-2478. Sonar : remove temporary variable in XceiverClientGrpc sendCommand (#165)
add 49dbb18 HDDS-2480. Sonar : remove log spam for exceptions inside XceiverClientGrpc reconnect (#170)
add 287b322 HDDS-2479. Sonar : replace instanceof with catch block in XceiverClientGrpc sendCommandWithRetry (#168)
add 8a11fe3 HDDS-2450 Datanode ReplicateContainer thread pool should be configurable (#134)
add d0fd848 HDDS-2473. Fix code reliability issues found by Sonar in Ozone Recon module. (#162)
add 0d86870 HDDS-2481. Close streams in TarContainerPacker (#167)
add 89d11ad HDDS-2034. Async RATIS pipeline creation and destroy through heartbeat commands (#29)
add 0e403a5 HDDS-2482. Enable github worflow actions full pull requests (#171)
add 37dc306 HDDS-2494 Sonar BigDecimal Cleanup (#175)
add 3869296 HDDS-2487. Ensure streams are closed (#173)
add 3ca37b1 HDDS-2492 Fix test clean up issue in TestSCMPipelineManager. (#179)
add a1ea8a4 HDDS-2468. scmcli close pipeline command not working
add d05ff4e HDDS-2471. Improve exception message for CompleteMultipartUpload
add aa828b1 HDDS-2470. Add partName, partNumber for CommitMultipartUpload
add 5264882 HDDS-2495. Sonar - "notify" may not wake up the appropriate thread
add 43be5c8 HDDS-2472. Use try-with-resources while creating FlushOptions in RDBStore (#161)
add 2f94fec Fix sonar warnings in HddsUtils (#184)
add bd0d873 HDDS-2418 Add the list trash command to the server side handling (#143)
add 9f57785 HDDS-2375. Refactor BlockOutputStream to allow flexible buffering. (#97)
add faedad7 HDDS-2500. Avoid fall-through in CloseContainerCommandHandler (#186)
add baf6cef HDDS-2515. no need of toString since log formatter can intatnally handles that. (#190)
add 6fd2ef0 HDDS-2511. Fix Sonar issues in OzoneManagerServiceProviderImpl. (#188)
add 5d51385 HDDS-2507. Remove the hard-coded exclusion of TestMiniChaosOzoneCluster (#182)
add ee6f22d HDDS-2502. Close ScmClient in RatisInsight. (#199)
add 79e3ff9 HDDS-2405. int2ByteString unnecessary byte array allocation (#197)
add 493a916 HDDS-2501. Sonar: Fix issues found in the ObjectEndpoint class. (#198)
add dba972d HDDS-2513. Removed unused private field (#187)
add b01c7c9 HDDS-2461. Logging by ChunkUtils is misleading (#144)
add 1b72718 HDDS-2503. Close FlushOptions in RDBStore (#181)
add a9a9f4f HDDS-2489. Change anonymous class based initialization in HddsUtils. (#172)
add 8af5ab8 HDDS-2525. Sonar : replace lambda with method reference in SCM BufferPool. (#210)
add 89bdb6a HDDS-2521. Multipart upload failing with NPE
add 6be3c84 HDDS-2524. Sonar : clumsy error handling in BlockOutputStream validateResponse. (#209)
add 4ba4065 HDDS-2526. Sonar : use format specifiers in Log inside HddsConfServlet (#211)
add b411439 HDDS-2520. Sonar: Avoid temporary variable scmSecurityClient (#208)
add d52fb26 HDDS-2488. Not enough arguments for log messages in GrpcXceiverService. (#191)
add 0b5df11 HDDS-2499. IsLeader information is lost when update pipeline state. (#180)
add ecb5bf4 HDDS-2490. Fixing sonarcloud errors. (#217)
add 4a4b03c HDDS-2527. Sonar: remove redundant temporary assignment in HddsVersionProvider (#219)
add b8819c3 HDDS-2517. Immediately return rather than holding to variable and then returning (#192)
add 287f9c1 HDDS-2514. removed unused method param (#189)
add ac59c4f HDDS-2486. Sonar: Avoid empty test methods (#220)
add e5a3b0c HDDS-2509. Code cleanup in replication package (#185)
add dcfe5f3 Revert "HDDS-2034. Async RATIS pipeline creation and destroy through heartbeat commands (#29)"
add 87d5a5a HDDS-2533. Disable failing acceptance and unit tests
add 814cb72 HDDS-2547. Sonar: remove volatile keyword from BlockOutputStream bloc… (#229)
add c3b14f6 HDDS-2442. Added support for service name in OM for CSR (#177)
add 83c66a7 HDDS-2548. Refactored return type as interface rather then impl. (#228)
add cb7e146 HDDS-2546. Reorder the modifiers to comply (#227)
add eda4575 HDDS-2544. Refactored for unnecessary toString (#225)
add 1d66049 HDDS-2543. Refactored for log format. (#224)
add 6186cf9 HDDS-2448 Delete container command should used a thread pool (#142)
add da36b09 HDDS-2550. Sonar: OzoneClient should be closed in GetAclKeyHandler (#231)
add d7ef779 HDDS-2545. Remove empty statement (#226)
add ba76d23 HDDS-2549. Invoke method(s) only conditionally (#230)
add b10ac6b HDDS-2535. TestOzoneManagerDoubleBufferWithOMResponse is flaky. (#216)
add 8ad88ce HDDS-2516. Code cleanup in EventQueue (#196)
add 2fea0af HDDS-2467. Allow running Freon validators with limited memory (#152)
add f29de78 HDDS-2247. Delete FileEncryptionInfo from KeyInfo when a Key is deleted (#200)
add e68b113 HDDS-2493. Sonar: Locking on a parameter in NetUtils.removeOutscope. (#174)
add 2134dec HDDS-2485. Disable XML external entity processing
add 92dcfdb HDDS-2580. Ensure resources are closed in Get/PutKeyHandler (#235)
add b1ac520 HDDS-2498. Fix sonar issues found in StorageContainerManager. (#178)
add e47acd1 HDDS-2512 Sonar TraceAllMethod NPE Could be Thrown (#193)
add 3c334f6 HDDS-2241. Optimize the refresh pipeline logic used by KeyManagerImpl… (#194)
add bdc11fc HDDS-2523. BufferPool.releaseBuffer may release a buffer different than the head of the list (#232)
add 26ffa6b HDDS-2522. Fix TestSecureOzoneCluster (#207)
add e7fd407 HDDS-2598. Remove unused private field "LOG" (#241)
add fab0242 HDDS-2597. Remove toString() as log calls it implicitly (#240)
add b557aec HDDS-2594. S3 RangeReads failing with NumberFormatException. (#242)
add bcb13d4 HDDS-2538. Fix issues found in DatabaseHelper. (#221)
add d29838b HDDS-2536. Add ozone.om.internal.service.id to OM HA configuration. (#218)
add 7d35218 HDDS-2394. Ozone S3 Gateway allows bucket name with underscore to be created (#243)
add 571d4a3 HDDS-2605. Use LongSupplier to avoid boxing (#256)
add a731eea HDDS-2603. Avoid unnecessary boxing in XceiverClientReply (#255)
add a2a19f7 HDDS-2300. Publish normalized Ratis metrics via the prometheus endpoint (#127)
add 6105e31 HDDS-2587. Enable sonarcloud measurement as part of CI builds (#253)
new 0c97903 Merged master into branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/bin/qbt => .asf.yaml | 18 +-
.../buildenv/Dockerfile | 8 +-
.../buildenv/entrypoint.sh | 6 +-
.github/workflows/post-commit.yml | 120 +++++++
.github/workflows/pr.yml | 114 ++++++
CONTRIBUTION.md | 5 +-
README.md | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 51 ++-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 3 +-
.../hdds/scm/client/ContainerOperationClient.java | 76 +---
.../hadoop/hdds/scm/client/HddsClientUtils.java | 42 +--
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 53 ++-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 21 +-
.../hadoop/hdds/scm/storage/CommitWatcher.java | 21 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 8 +-
.../hadoop/hdds/cli/HddsVersionProvider.java | 3 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 12 +-
.../hadoop/hdds/client/ContainerBlockID.java | 13 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 2 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 4 +-
.../hdds/protocolPB/SCMSecurityProtocolPB.java | 5 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 3 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 12 -
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 76 ++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientReply.java | 18 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 9 -
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 10 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 27 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 55 ++-
.../scm/protocol/ScmBlockLocationProtocol.java | 5 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +-
.../scm/protocolPB/ScmBlockLocationProtocolPB.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 37 +-
.../StorageContainerLocationProtocolPB.java | 4 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 17 +-
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 5 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 64 ++--
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 2 +-
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 75 ++++
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 61 ++--
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 18 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../org/apache/hadoop/ozone/audit/SCMAction.java | 1 +
.../org/apache/hadoop/ozone/common/Checksum.java | 37 +-
.../apache/hadoop/ozone/common/ChecksumData.java | 28 +-
.../apache/hadoop/ozone/common/ChunkBuffer.java | 87 +++++
.../common/ChunkBufferImplWithByteBuffer.java | 108 ++++++
.../ozone/container/common/helpers/BlockData.java | 32 +-
.../java/org/apache/hadoop/ozone/lease/Lease.java | 46 ++-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 5 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 20 +-
.../apache/hadoop/ozone/lease/package-info.java | 6 +-
.../proto/StorageContainerLocationProtocol.proto | 30 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 12 +-
.../common/src/main/resources/ozone-default.xml | 47 +--
.../ratis/TestContainerCommandRequestMessage.java | 8 +-
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 151 ++++----
.../certificates/TestCertificateSignRequest.java | 33 ++
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 53 ++-
.../hdds/utils/db/TestTypedRDBTableStore.java | 4 -
.../hadoop/ozone/common/TestChunkBuffer.java | 134 +++++++
.../hadoop/ozone/lease/TestLeaseManager.java | 7 +-
.../apache/hadoop/ozone/lease/package-info.java | 8 +-
.../src/test/resources/log4j.properties | 0
.../{good.xml => external-entity.xml} | 7 +-
.../hadoop/hdds/conf/ConfigFileAppender.java | 21 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 18 +-
.../container/common/impl/ContainerDataYaml.java | 5 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../ozone/container/common/interfaces/Handler.java | 60 ++--
.../common/report/PipelineReportPublisher.java | 4 +-
.../container/common/report/ReportPublisher.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 91 +++++
.../common/statemachine/DatanodeStateMachine.java | 16 +-
.../CloseContainerCommandHandler.java | 3 +-
.../commandhandler/CommandDispatcher.java | 6 +
.../commandhandler/CommandHandler.java | 8 +
.../DeleteContainerCommandHandler.java | 70 +++-
.../ReplicateContainerCommandHandler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 8 +-
.../server/ratis/ContainerStateMachine.java | 16 +-
.../transport/server/ratis/XceiverServerRatis.java | 27 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 12 +-
.../container/keyvalue/TarContainerPacker.java | 170 ++++-----
.../container/keyvalue/helpers/ChunkUtils.java | 55 +--
.../keyvalue/impl/ChunkManagerDummyImpl.java | 11 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 5 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 59 ++--
.../replication/ContainerStreamingOutput.java | 5 +-
.../container/replication/GrpcOutputStream.java | 129 +++++++
.../replication/GrpcReplicationClient.java | 24 +-
.../replication/GrpcReplicationService.java | 94 +----
.../OnDemandContainerReplicationSource.java | 13 +-
.../replication/ReplicationSupervisor.java | 61 +++-
.../protocol/StorageContainerDatanodeProtocol.java | 5 +-
.../StorageContainerDatanodeProtocolPB.java | 4 +-
.../proto/StorageContainerDatanodeProtocol.proto | 1 +
.../container/common/impl/TestHddsDispatcher.java | 16 +-
.../container/common/interfaces/TestHandler.java | 8 +-
.../container/keyvalue/TestKeyValueHandler.java | 10 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 7 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 19 +
.../replication/TestGrpcOutputStream.java | 213 ++++++++++++
.../replication/TestReplicationSupervisor.java | 218 +++++++++---
hadoop-hdds/framework/pom.xml | 13 +-
.../hadoop/hdds/server/PrometheusServlet.java | 13 +-
.../hadoop/hdds/server/RatisDropwizardExports.java | 31 +-
.../hdds/server/RatisNameRewriteSampleBuilder.java | 104 ++++++
.../hadoop/hdds/server/events/EventExecutor.java | 2 +-
.../hadoop/hdds/server/events/EventHandler.java | 4 +-
.../hadoop/hdds/server/events/EventPublisher.java | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 13 +-
.../hadoop/hdds/server/events/EventWatcher.java | 46 +--
.../hdds/server/events/EventWatcherMetrics.java | 11 +-
.../hdds/server/events/SingleThreadExecutor.java | 36 +-
.../hadoop/hdds/server/events/TypedEvent.java | 4 +-
.../hadoop/hdds/server/events/package-info.java | 7 +-
.../hdds/server/TestRatisDropwizardExports.java | 66 ++++
.../hadoop/hdds/server/TestRatisNameRewrite.java | 107 ++++++
.../hdds/server/events/EventHandlerStub.java | 4 +-
hadoop-hdds/pom.xml | 3 +-
.../hdds/scm/container/ReplicationManager.java | 4 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 50 +--
.../hdds/scm/pipeline/PipelineStateManager.java | 8 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 13 +-
...inerLocationProtocolServerSideTranslatorPB.java | 35 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 75 ++--
.../hadoop/hdds/scm/safemode/SafeModeHandler.java | 5 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 49 ++-
.../hdds/scm/server/SCMHTTPServerConfig.java | 82 +++++
.../hdds/scm/server/SCMSecurityProtocolServer.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 88 +++--
.../server/StorageContainerManagerHttpServer.java | 6 +-
.../scm/server/StorageContainerManagerStarter.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 14 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 2 +
.../scm/pipeline/MockRatisPipelineProvider.java | 15 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 3 +-
.../TestOneReplicaPipelineSafeModeRule.java | 3 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +-
.../ozone/container/replication/package-info.java | 18 -
.../ozone/client/protocol/ClientProtocol.java | 20 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 50 ++-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 24 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 3 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 18 +
.../hadoop/ozone/om/exceptions/OMException.java | 12 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 11 +
.../om/helpers/OmMultipartUploadCompleteList.java | 8 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 19 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 42 +++
.../src/main/proto/OzoneManagerProtocol.proto | 26 ++
hadoop-ozone/dev-support/checks/integration.sh | 3 +-
hadoop-ozone/dev-support/checks/shellcheck.sh | 1 -
.../dev-support/checks/{build.sh => sonar.sh} | 10 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 4 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 4 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 30 +-
.../main/compose/ozone-om-ha/{test.sh => run.sh} | 0
.../src/main/compose/ozoneperf/docker-compose.yaml | 4 +
.../dist/src/main/compose/ozonescripts/Dockerfile | 6 +-
.../dist/src/main/compose/ozonescripts/README.md | 6 +-
.../main/compose/ozonescripts/docker-compose.yaml | 18 +-
hadoop-ozone/dist/src/main/docker/Dockerfile | 2 +
.../{dockerbin => dockerlibexec}/entrypoint.sh | 0
.../main/{dockerbin => dockerlibexec}/envtoconf.py | 0
.../{dockerbin => dockerlibexec}/transformation.py | 0
.../src/main/smoketest/basic/ozone-shell.robot | 6 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 14 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 31 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 4 +
.../apache/hadoop/ozone/insight/LogSubcommand.java | 2 +-
.../ozone/insight/datanode/RatisInsight.java | 32 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 151 ++++----
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 58 +++-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 6 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 385 +++++++++------------
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 19 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 82 +++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 11 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../container/common/helpers/TestBlockData.java | 12 +
.../container/metrics/TestContainerMetrics.java | 9 +-
.../container/server/TestContainerServer.java | 7 +-
.../server/TestSecureContainerServer.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 133 ++++++-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 3 +
.../org/apache/hadoop/ozone/om/KeyManager.java | 19 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 206 ++++-------
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 9 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 45 +++
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 21 +-
.../S3MultipartUploadCommitPartRequest.java | 23 +-
.../S3MultipartUploadCompleteRequest.java | 241 +++++++------
.../S3MultipartUploadCompleteResponse.java | 29 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 36 +-
.../hadoop/ozone/web/ozShell/OzoneShell.java | 2 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 44 +--
.../web/ozShell/bucket/CreateBucketHandler.java | 64 ++--
.../web/ozShell/bucket/DeleteBucketHandler.java | 20 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 37 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 22 +-
.../web/ozShell/bucket/ListBucketHandler.java | 44 +--
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 43 +--
.../ozone/web/ozShell/bucket/S3BucketMapping.java | 26 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 35 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 59 ++--
.../ozone/web/ozShell/keys/DeleteKeyHandler.java | 25 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 44 +--
.../ozone/web/ozShell/keys/GetKeyHandler.java | 80 +++--
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 39 ++-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 76 ++--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 92 ++---
.../web/ozShell/keys/RemoveAclKeyHandler.java | 59 ++--
.../ozone/web/ozShell/keys/RenameKeyHandler.java | 26 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 39 ++-
.../ozone/web/ozShell/s3/GetS3SecretHandler.java | 25 +-
.../hadoop/ozone/web/ozShell/s3/S3Shell.java | 5 +-
.../web/ozShell/token/CancelTokenHandler.java | 34 +-
.../ozone/web/ozShell/token/GetTokenHandler.java | 38 +-
.../ozone/web/ozShell/token/RenewTokenHandler.java | 38 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 37 +-
.../web/ozShell/volume/CreateVolumeHandler.java | 55 +--
.../web/ozShell/volume/DeleteVolumeHandler.java | 15 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 30 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 11 +-
.../web/ozShell/volume/ListVolumeHandler.java | 62 ++--
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 37 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 43 +--
.../web/ozShell/volume/UpdateVolumeHandler.java | 22 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 13 +-
hadoop-ozone/ozonefs-lib-current/pom.xml | 15 +
hadoop-ozone/pom.xml | 3 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 22 +-
.../ozone/recon/api/ContainerKeyService.java | 22 +-
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 6 +-
.../recon/spi/OzoneManagerServiceProvider.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 30 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 20 +-
.../endpoint/MultiDeleteRequestUnmarshaller.java | 8 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 244 +++++++------
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 4 +
.../ozone/s3/util/RangeHeaderParserUtil.java | 4 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 26 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 8 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 8 +-
.../ozone/s3/util/TestRangeHeaderParserUtil.java | 6 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 157 ++++-----
.../audit/parser/handler/LoadCommandHandler.java | 11 +-
.../audit/parser/handler/QueryCommandHandler.java | 11 +-
.../parser/handler/TemplateCommandHandler.java | 11 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 10 +-
.../hadoop/ozone/freon/ContentGenerator.java | 6 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 9 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 15 +-
.../ozone/freon/OzoneClientKeyValidator.java | 93 ++++-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 33 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 4 +-
.../apache/hadoop/ozone/freon/SameKeyReader.java | 65 +---
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 82 +++--
.../ozone/genesis/BenchmarkBlockDataToString.java | 166 +++++++++
.../org/apache/hadoop/ozone/genesis/Genesis.java | 5 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 24 +-
pom.xml | 21 +-
274 files changed, 5770 insertions(+), 3131 deletions(-)
copy dev-support/bin/qbt => .asf.yaml (71%)
mode change 100755 => 100644
copy hadoop-hdds/container-service/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider => .github/buildenv/Dockerfile (83%)
copy hadoop-hdds/container-service/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider => .github/buildenv/entrypoint.sh (85%)
mode change 100644 => 100755
create mode 100644 .github/workflows/post-commit.yml
create mode 100644 .github/workflows/pr.yml
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RDBMetrics.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChunkBuffer.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChunkBufferImplWithByteBuffer.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChunkBuffer.java
copy hadoop-hdds/{container-service => common}/src/test/resources/log4j.properties (100%)
copy hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/{good.xml => external-entity.xml} (93%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcOutputStream.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcOutputStream.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/RatisDropwizardExports.java (58%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/RatisNameRewriteSampleBuilder.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestRatisDropwizardExports.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestRatisNameRewrite.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
copy hadoop-ozone/dev-support/checks/{build.sh => sonar.sh} (68%)
rename hadoop-ozone/dist/src/main/compose/ozone-om-ha/{test.sh => run.sh} (100%)
rename hadoop-ozone/dist/src/main/{dockerbin => dockerlibexec}/entrypoint.sh (100%)
rename hadoop-ozone/dist/src/main/{dockerbin => dockerlibexec}/envtoconf.py (100%)
rename hadoop-ozone/dist/src/main/{dockerbin => dockerlibexec}/transformation.py (100%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkBlockDataToString.java
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: Merged master into branch
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 0c979038d5af35769ea0faf397f6db565134995c
Merge: 88147d0 6105e31
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Fri Nov 22 18:30:56 2019 +0000
Merged master into branch
.../dev-support/checks/unit.sh => .asf.yaml | 31 +-
.../checks/unit.sh => .github/buildenv/Dockerfile | 23 +-
.../unit.sh => .github/buildenv/entrypoint.sh | 21 +-
.github/workflows/post-commit.yml | 120 +++++++
.github/workflows/pr.yml | 114 ++++++
CONTRIBUTION.md | 5 +-
README.md | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 51 ++-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 3 +-
.../hdds/scm/client/ContainerOperationClient.java | 76 +---
.../hadoop/hdds/scm/client/HddsClientUtils.java | 42 +--
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 53 ++-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 21 +-
.../hadoop/hdds/scm/storage/CommitWatcher.java | 21 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 8 +-
.../hadoop/hdds/cli/HddsVersionProvider.java | 3 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 12 +-
.../hadoop/hdds/client/ContainerBlockID.java | 13 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 2 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 4 +-
.../hdds/protocolPB/SCMSecurityProtocolPB.java | 5 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 3 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 12 -
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 76 ++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientReply.java | 18 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 9 -
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 10 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 27 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 55 ++-
.../scm/protocol/ScmBlockLocationProtocol.java | 5 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +-
.../scm/protocolPB/ScmBlockLocationProtocolPB.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 37 +-
.../StorageContainerLocationProtocolPB.java | 4 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 17 +-
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 5 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 64 ++--
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 2 +-
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 75 ++++
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 61 ++--
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 18 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../org/apache/hadoop/ozone/audit/SCMAction.java | 1 +
.../org/apache/hadoop/ozone/common/Checksum.java | 37 +-
.../apache/hadoop/ozone/common/ChecksumData.java | 28 +-
.../apache/hadoop/ozone/common/ChunkBuffer.java | 87 +++++
.../common/ChunkBufferImplWithByteBuffer.java | 108 ++++++
.../ozone/container/common/helpers/BlockData.java | 32 +-
.../java/org/apache/hadoop/ozone/lease/Lease.java | 46 ++-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 5 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 20 +-
.../apache/hadoop/ozone/lease/package-info.java | 6 +-
.../proto/StorageContainerLocationProtocol.proto | 30 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 12 +-
.../common/src/main/resources/ozone-default.xml | 47 +--
.../ratis/TestContainerCommandRequestMessage.java | 8 +-
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 151 ++++----
.../certificates/TestCertificateSignRequest.java | 33 ++
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 53 ++-
.../hdds/utils/db/TestTypedRDBTableStore.java | 4 -
.../hadoop/ozone/common/TestChunkBuffer.java | 134 +++++++
.../hadoop/ozone/lease/TestLeaseManager.java | 7 +-
.../apache/hadoop/ozone/lease/package-info.java | 8 +-
.../common/src/test/resources/log4j.properties | 23 ++
.../networkTopologyTestFiles/external-entity.xml | 52 +++
.../hadoop/hdds/conf/ConfigFileAppender.java | 21 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 18 +-
.../container/common/impl/ContainerDataYaml.java | 5 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../ozone/container/common/interfaces/Handler.java | 60 ++--
.../common/report/PipelineReportPublisher.java | 4 +-
.../container/common/report/ReportPublisher.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 91 +++++
.../common/statemachine/DatanodeStateMachine.java | 16 +-
.../CloseContainerCommandHandler.java | 3 +-
.../commandhandler/CommandDispatcher.java | 6 +
.../commandhandler/CommandHandler.java | 8 +
.../DeleteContainerCommandHandler.java | 70 +++-
.../ReplicateContainerCommandHandler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 8 +-
.../server/ratis/ContainerStateMachine.java | 16 +-
.../transport/server/ratis/XceiverServerRatis.java | 27 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 12 +-
.../container/keyvalue/TarContainerPacker.java | 170 ++++-----
.../container/keyvalue/helpers/ChunkUtils.java | 55 +--
.../keyvalue/impl/ChunkManagerDummyImpl.java | 11 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 5 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 59 ++--
.../replication/ContainerStreamingOutput.java | 5 +-
.../container/replication/GrpcOutputStream.java | 129 +++++++
.../replication/GrpcReplicationClient.java | 24 +-
.../replication/GrpcReplicationService.java | 94 +----
.../OnDemandContainerReplicationSource.java | 13 +-
.../replication/ReplicationSupervisor.java | 61 +++-
.../protocol/StorageContainerDatanodeProtocol.java | 5 +-
.../StorageContainerDatanodeProtocolPB.java | 4 +-
.../proto/StorageContainerDatanodeProtocol.proto | 1 +
.../container/common/impl/TestHddsDispatcher.java | 16 +-
.../container/common/interfaces/TestHandler.java | 8 +-
.../container/keyvalue/TestKeyValueHandler.java | 10 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 7 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 19 +
.../replication/TestGrpcOutputStream.java | 213 ++++++++++++
.../replication/TestReplicationSupervisor.java | 218 +++++++++---
hadoop-hdds/framework/pom.xml | 13 +-
.../hadoop/hdds/server/PrometheusServlet.java | 13 +-
.../hadoop/hdds/server/RatisDropwizardExports.java | 31 +-
.../hdds/server/RatisNameRewriteSampleBuilder.java | 104 ++++++
.../hadoop/hdds/server/events/EventExecutor.java | 2 +-
.../hadoop/hdds/server/events/EventHandler.java | 4 +-
.../hadoop/hdds/server/events/EventPublisher.java | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 13 +-
.../hadoop/hdds/server/events/EventWatcher.java | 46 +--
.../hdds/server/events/EventWatcherMetrics.java | 11 +-
.../hdds/server/events/SingleThreadExecutor.java | 36 +-
.../hadoop/hdds/server/events/TypedEvent.java | 4 +-
.../hadoop/hdds/server/events/package-info.java | 7 +-
.../hdds/server/TestRatisDropwizardExports.java | 66 ++++
.../hadoop/hdds/server/TestRatisNameRewrite.java | 107 ++++++
.../hdds/server/events/EventHandlerStub.java | 4 +-
hadoop-hdds/pom.xml | 3 +-
.../hdds/scm/container/ReplicationManager.java | 4 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 50 +--
.../hdds/scm/pipeline/PipelineStateManager.java | 8 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 13 +-
...inerLocationProtocolServerSideTranslatorPB.java | 35 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 75 ++--
.../hadoop/hdds/scm/safemode/SafeModeHandler.java | 5 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 49 ++-
.../hdds/scm/server/SCMHTTPServerConfig.java | 82 +++++
.../hdds/scm/server/SCMSecurityProtocolServer.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 88 +++--
.../server/StorageContainerManagerHttpServer.java | 6 +-
.../scm/server/StorageContainerManagerStarter.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 14 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 2 +
.../scm/pipeline/MockRatisPipelineProvider.java | 15 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 3 +-
.../TestOneReplicaPipelineSafeModeRule.java | 3 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +-
.../ozone/container/replication/package-info.java | 18 -
.../ozone/client/protocol/ClientProtocol.java | 20 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 50 ++-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 24 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 3 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 18 +
.../hadoop/ozone/om/exceptions/OMException.java | 12 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 11 +
.../om/helpers/OmMultipartUploadCompleteList.java | 8 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 19 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 42 +++
.../src/main/proto/OzoneManagerProtocol.proto | 26 ++
hadoop-ozone/dev-support/checks/integration.sh | 3 +-
hadoop-ozone/dev-support/checks/shellcheck.sh | 1 -
.../dev-support/checks/{unit.sh => sonar.sh} | 18 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 4 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 4 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 30 +-
.../main/compose/ozone-om-ha/{test.sh => run.sh} | 0
.../src/main/compose/ozoneperf/docker-compose.yaml | 4 +
.../dist/src/main/compose/ozonescripts/Dockerfile | 6 +-
.../dist/src/main/compose/ozonescripts/README.md | 6 +-
.../main/compose/ozonescripts/docker-compose.yaml | 18 +-
hadoop-ozone/dist/src/main/docker/Dockerfile | 2 +
.../{dockerbin => dockerlibexec}/entrypoint.sh | 0
.../main/{dockerbin => dockerlibexec}/envtoconf.py | 0
.../{dockerbin => dockerlibexec}/transformation.py | 0
.../src/main/smoketest/basic/ozone-shell.robot | 6 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 14 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 31 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 4 +
.../apache/hadoop/ozone/insight/LogSubcommand.java | 2 +-
.../ozone/insight/datanode/RatisInsight.java | 32 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 151 ++++----
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 58 +++-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 6 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 385 +++++++++------------
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 19 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 82 +++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 11 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../container/common/helpers/TestBlockData.java | 12 +
.../container/metrics/TestContainerMetrics.java | 9 +-
.../container/server/TestContainerServer.java | 7 +-
.../server/TestSecureContainerServer.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 133 ++++++-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 3 +
.../org/apache/hadoop/ozone/om/KeyManager.java | 19 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 206 ++++-------
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 9 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 45 +++
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 21 +-
.../S3MultipartUploadCommitPartRequest.java | 23 +-
.../S3MultipartUploadCompleteRequest.java | 241 +++++++------
.../S3MultipartUploadCompleteResponse.java | 29 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 36 +-
.../hadoop/ozone/web/ozShell/OzoneShell.java | 2 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 44 +--
.../web/ozShell/bucket/CreateBucketHandler.java | 64 ++--
.../web/ozShell/bucket/DeleteBucketHandler.java | 20 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 37 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 22 +-
.../web/ozShell/bucket/ListBucketHandler.java | 44 +--
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 43 +--
.../ozone/web/ozShell/bucket/S3BucketMapping.java | 26 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 35 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 59 ++--
.../ozone/web/ozShell/keys/DeleteKeyHandler.java | 25 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 44 +--
.../ozone/web/ozShell/keys/GetKeyHandler.java | 80 +++--
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 39 ++-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 76 ++--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 92 ++---
.../web/ozShell/keys/RemoveAclKeyHandler.java | 59 ++--
.../ozone/web/ozShell/keys/RenameKeyHandler.java | 26 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 39 ++-
.../ozone/web/ozShell/s3/GetS3SecretHandler.java | 25 +-
.../hadoop/ozone/web/ozShell/s3/S3Shell.java | 5 +-
.../web/ozShell/token/CancelTokenHandler.java | 34 +-
.../ozone/web/ozShell/token/GetTokenHandler.java | 38 +-
.../ozone/web/ozShell/token/RenewTokenHandler.java | 38 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 37 +-
.../web/ozShell/volume/CreateVolumeHandler.java | 55 +--
.../web/ozShell/volume/DeleteVolumeHandler.java | 15 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 30 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 11 +-
.../web/ozShell/volume/ListVolumeHandler.java | 62 ++--
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 37 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 43 +--
.../web/ozShell/volume/UpdateVolumeHandler.java | 22 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 13 +-
hadoop-ozone/ozonefs-lib-current/pom.xml | 15 +
hadoop-ozone/pom.xml | 3 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 22 +-
.../ozone/recon/api/ContainerKeyService.java | 22 +-
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 6 +-
.../recon/spi/OzoneManagerServiceProvider.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 30 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 20 +-
.../endpoint/MultiDeleteRequestUnmarshaller.java | 8 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 244 +++++++------
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 4 +
.../ozone/s3/util/RangeHeaderParserUtil.java | 4 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 26 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 8 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 8 +-
.../ozone/s3/util/TestRangeHeaderParserUtil.java | 6 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 157 ++++-----
.../audit/parser/handler/LoadCommandHandler.java | 11 +-
.../audit/parser/handler/QueryCommandHandler.java | 11 +-
.../parser/handler/TemplateCommandHandler.java | 11 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 10 +-
.../hadoop/ozone/freon/ContentGenerator.java | 6 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 9 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 15 +-
.../ozone/freon/OzoneClientKeyValidator.java | 93 ++++-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 33 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 4 +-
.../apache/hadoop/ozone/freon/SameKeyReader.java | 65 +---
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 82 +++--
.../ozone/genesis/BenchmarkBlockDataToString.java | 166 +++++++++
.../org/apache/hadoop/ozone/genesis/Genesis.java | 5 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 24 +-
pom.xml | 21 +-
274 files changed, 5840 insertions(+), 3180 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 8b37c57,365750a..2addbc4
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@@ -105,36 -104,20 +104,30 @@@ public interface StorageContainerLocati
void deleteContainer(long containerID) throws IOException;
/**
- * Queries a list of Node Statuses.
- * @param state
+ * Queries a list of Node Statuses. Passing a null for either opState or
+ * state acts like a wildcard returning all nodes in that state.
+ * @param opState The node operational state
+ * @param state The node health
* @return List of Datanodes.
*/
- List<HddsProtos.Node> queryNode(HddsProtos.NodeState state,
- HddsProtos.QueryScope queryScope, String poolName) throws IOException;
+ List<HddsProtos.Node> queryNode(HddsProtos.NodeOperationalState opState,
+ HddsProtos.NodeState state, HddsProtos.QueryScope queryScope,
+ String poolName) throws IOException;
+
+ void decommissionNodes(List<String> nodes) throws IOException;
+
+ void recommissionNodes(List<String> nodes) throws IOException;
+
+ void startMaintenanceNodes(List<String> nodes, int endInHours)
+ throws IOException;
/**
- * Notify from client when begin or finish creating objects like pipeline
- * or containers on datanodes.
- * Container will be in Operational state after that.
- * @param type object type
- * @param id object id
- * @param op operation type (e.g., create, close, delete)
- * @param stage creation stage
+ * Close a container.
+ *
+ * @param containerID ID of the container to close
+ * @throws IOException in case of any Exception
*/
- void notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type type, long id,
- ObjectStageChangeRequestProto.Op op,
- ObjectStageChangeRequestProto.Stage stage) throws IOException;
+ void closeContainer(long containerID) throws IOException;
/**
* Creates a replication pipeline of a specified type.
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index 1c32380,8110242..2e6e2c4
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -2483,14 -2466,11 +2466,22 @@@
</description>
</property>
<property>
+ <name>ozone.scm.datanode.admin.monitor.interval</name>
+ <value>30s</value>
+ <tag>SCM</tag>
+ <description>
+ This sets how frequently the datanode admin monitor runs to check for
+ nodes added to the admin workflow or removed from it. The progress
+ of decommissioning and entering maintenance nodes is also checked to see
+ if they have completed.
+ </description>
+ </property>
++ <property>
+ <name>ozone.client.list.trash.keys.max</name>
+ <value>1000</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>
+ The maximum number of keys to return for a list trash request.
+ </description>
+ </property>
</configuration>
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 47e2c92,f48fad1..37da3c5
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@@ -334,14 -308,14 +348,13 @@@ public final class StorageContainerLoca
return NodeQueryResponseProto.newBuilder()
.addAllDatanodes(datanodes)
.build();
-
}
- public ObjectStageChangeResponseProto notifyObjectStageChange(
- ObjectStageChangeRequestProto request)
+ public SCMCloseContainerResponseProto closeContainer(
+ SCMCloseContainerRequestProto request)
throws IOException {
- impl.notifyObjectStageChange(request.getType(), request.getId(),
- request.getOp(), request.getStage());
- return ObjectStageChangeResponseProto.newBuilder().build();
+ impl.closeContainer(request.getContainerID());
+ return SCMCloseContainerResponseProto.newBuilder().build();
}
public ListPipelineResponseProto listPipelines(
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index d558162,80a6a07..f1a6b22
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@@ -34,7 -35,7 +34,8 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+ import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck;
import org.apache.hadoop.hdds.scm.container.ContainerID;
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org