You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2026/07/30 06:36:17 UTC
(ozone) branch master updated (8acc0413a95 -> ec381e5d3b0)
This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 8acc0413a95 HDDS-14812. Support datanode topology in MiniOzoneCluster (#10598)
add 147b009a6db HDDS-11622. Support domain socket creation (#7397)
add 46b858007e6 HDDS-11627. Support getBlock operation on short-circuit channel. (#7456)
add fb9398a3131 HDDS-11626. Document short-circuit read feature (#7529)
add 13a905468a2 Merge remote-tracking branch 'origin/master' into HDDS-10685
add cd3e1c7ca04 HDDS-12883. Close file descriptor of block file in datanode (#8319)
add 1395fae0640 Merge remote-tracking branch 'origin/master' into HDDS-10685
add 625a6ad85cd HDDS-11651. Support short-circuit read channel for freon rk command (#8316)
add 5676ac4b338 HDDS-15318. Download libhadoop library during maven build (#10305)
add 879707874bc Merge remote-tracking branch 'origin/master' into HDDS-10685
add 2f90c1731a3 HDDS-15404. Enable short-circuit unit test in CI (#10377)
add 6c9676481b9 HDDS-15442. Enhance short-circuit robot test (#10400)
add c2118e70585 Merge branch 'master' of github.com:apache/ozone into HDDS-10685
add 277737474cd HDDS-15559. Add libhadoop to DYLD_LIBRARY_PATH or LD_LIBRARY_PATH (#10513)
add 1ea7e9e7985 Merge remote-tracking branch 'origin/master' into HDDS-10685
add b01e0c715fa HDDS-15992. Convert short-circuit-read design PDF to markdown (#10877)
add 6db4545f1da HDDS-16013. Remove HDDSLayoutFeature.SHORT_CIRCUIT_READS layout version (#10893)
new ec381e5d3b0 Merge remote-tracking branch 'origin/HDDS-10685' into master
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 2 +-
hadoop-hdds/client/pom.xml | 4 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 52 ++
.../hadoop/hdds/scm/XceiverClientCreator.java | 37 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 19 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 104 ++-
.../hadoop/hdds/scm/XceiverClientShortCircuit.java | 623 ++++++++++++++++
.../hadoop/hdds/scm/storage/BlockInputStream.java | 149 +++-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/DomainPeer.java | 111 +++
.../hdds/scm/storage/DomainSocketFactory.java | 276 +++++++
.../hdds/scm/storage/LocalChunkInputStream.java | 116 +++
.../org/apache/hadoop/hdds/DatanodeVersion.java | 2 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 14 +
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 1 +
.../ContainerCommandResponseBuilders.java | 20 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 30 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 14 +
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 24 +-
.../common/src/main/resources/ozone-default.xml | 11 +-
.../hadoop/hdds/utils/TestProtobufUtils.java | 36 +
.../ozone/container/ContainerTestHelper.java | 2 +-
.../container/common/helpers/ContainerMetrics.java | 49 +-
.../ozone/container/common/interfaces/Handler.java | 23 +-
.../common/transport/server/Receiver.java | 353 +++++++++
.../server/XceiverServerDomainSocket.java | 324 +++++++++
.../common/transport/server/XceiverServerGrpc.java | 53 +-
.../common/transport/server/XceiverServerSpi.java | 2 +
.../transport/server/ratis/XceiverServerRatis.java | 5 +
.../ozone/container/keyvalue/KeyValueHandler.java | 53 +-
.../container/keyvalue/helpers/BlockUtils.java | 9 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 7 +
.../keyvalue/impl/FilePerBlockStrategy.java | 22 +-
.../keyvalue/interfaces/ChunkManager.java | 17 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 77 +-
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../design/short-circuit-read-current-flow.png | Bin 0 -> 33740 bytes
.../design/short-circuit-read-getblock-flow.png | Bin 0 -> 66181 bytes
.../design/short-circuit-read-hbase-benchmark.png | Bin 0 -> 34029 bytes
.../short-circuit-read-mmap-cache-benchmark.png | Bin 0 -> 28688 bytes
.../design/short-circuit-read-target-flow.png | Bin 0 -> 39362 bytes
.../short-circuit-read-unix-domain-socket.png | Bin 0 -> 82958 bytes
.../docs/content/design/short-circuit-read.md | 175 +++++
.../docs/content/feature/Short-Circuit-Read.md | 90 +++
.../docs/content/feature/Short-Circuit-Read.zh.md | 88 +++
.../src/main/proto/DatanodeClientProtocol.proto | 12 +
hadoop-hdds/rocks-native/pom.xml | 6 +-
.../ozone/client/MockXceiverClientFactory.java | 9 +-
.../client/checksum/FileChecksumBenchmark.java | 12 +
.../dist/dev-support/bin/dist-layout-stitching | 18 +
.../diskbalancer.yaml => ozone/short-circuit.yaml} | 11 +-
.../{test-failures1.sh => test-short-circuit.sh} | 17 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +
.../src/main/smoketest/freon/read-write-key.robot | 7 +
.../smoketest/short-circuit/short-circuit.robot | 55 ++
.../dist/src/shell/ozone/ozone-functions.sh | 42 ++
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +
hadoop-ozone/integration-test/pom.xml | 11 +
.../hadoop/hdds/TestXceiverServerDomainSocket.java | 793 +++++++++++++++++++++
.../hadoop/hdds/scm/TestXceiverClientManager.java | 6 +-
.../hdds/scm/TestXceiverClientManagerSC.java | 109 +++
.../TestDNDataDistributionFinalization.java | 11 +-
.../TestScmDataDistributionFinalization.java | 8 +-
.../ozone/client/rpc/read/InputStreamTests.java | 22 +-
.../client/rpc/read/TestChunkInputStream.java | 16 +-
.../client/rpc/read/TestDomainSocketFactory.java | 81 +++
.../client/rpc/read/TestLocalChunkInputStream.java | 163 +++++
.../ozone/om/TestOmContainerLocationCache.java | 4 +
.../ozone/om/service/TestBlockDeletionService.java | 7 +-
pom.xml | 183 +++++
72 files changed, 4493 insertions(+), 158 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientShortCircuit.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainPeer.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainSocketFactory.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/LocalChunkInputStream.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/Receiver.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-current-flow.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-getblock-flow.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-hbase-benchmark.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-mmap-cache-benchmark.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-target-flow.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-unix-domain-socket.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read.md
create mode 100644 hadoop-hdds/docs/content/feature/Short-Circuit-Read.md
create mode 100644 hadoop-hdds/docs/content/feature/Short-Circuit-Read.zh.md
copy hadoop-ozone/dist/src/main/compose/{ozonesecure/diskbalancer.yaml => ozone/short-circuit.yaml} (80%)
copy hadoop-ozone/dist/src/main/compose/ozone/{test-failures1.sh => test-short-circuit.sh} (79%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/short-circuit/short-circuit.robot
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/01: Merge remote-tracking branch 'origin/HDDS-10685' into master
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit ec381e5d3b00888583fe72130f26884f96eba161
Merge: 8acc0413a95 6db4545f1da
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Thu Jul 30 11:50:32 2026 +0800
Merge remote-tracking branch 'origin/HDDS-10685' into master
.github/workflows/ci.yml | 2 +-
hadoop-hdds/client/pom.xml | 4 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 52 ++
.../hadoop/hdds/scm/XceiverClientCreator.java | 37 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 19 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 104 ++-
.../hadoop/hdds/scm/XceiverClientShortCircuit.java | 623 ++++++++++++++++
.../hadoop/hdds/scm/storage/BlockInputStream.java | 149 +++-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/DomainPeer.java | 111 +++
.../hdds/scm/storage/DomainSocketFactory.java | 276 +++++++
.../hdds/scm/storage/LocalChunkInputStream.java | 116 +++
.../org/apache/hadoop/hdds/DatanodeVersion.java | 2 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 14 +
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 1 +
.../ContainerCommandResponseBuilders.java | 20 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 30 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 14 +
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 24 +-
.../common/src/main/resources/ozone-default.xml | 11 +-
.../hadoop/hdds/utils/TestProtobufUtils.java | 36 +
.../ozone/container/ContainerTestHelper.java | 2 +-
.../container/common/helpers/ContainerMetrics.java | 49 +-
.../ozone/container/common/interfaces/Handler.java | 23 +-
.../common/transport/server/Receiver.java | 353 +++++++++
.../server/XceiverServerDomainSocket.java | 324 +++++++++
.../common/transport/server/XceiverServerGrpc.java | 53 +-
.../common/transport/server/XceiverServerSpi.java | 2 +
.../transport/server/ratis/XceiverServerRatis.java | 5 +
.../ozone/container/keyvalue/KeyValueHandler.java | 53 +-
.../container/keyvalue/helpers/BlockUtils.java | 9 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 7 +
.../keyvalue/impl/FilePerBlockStrategy.java | 22 +-
.../keyvalue/interfaces/ChunkManager.java | 17 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 77 +-
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../design/short-circuit-read-current-flow.png | Bin 0 -> 33740 bytes
.../design/short-circuit-read-getblock-flow.png | Bin 0 -> 66181 bytes
.../design/short-circuit-read-hbase-benchmark.png | Bin 0 -> 34029 bytes
.../short-circuit-read-mmap-cache-benchmark.png | Bin 0 -> 28688 bytes
.../design/short-circuit-read-target-flow.png | Bin 0 -> 39362 bytes
.../short-circuit-read-unix-domain-socket.png | Bin 0 -> 82958 bytes
.../docs/content/design/short-circuit-read.md | 175 +++++
.../docs/content/feature/Short-Circuit-Read.md | 90 +++
.../docs/content/feature/Short-Circuit-Read.zh.md | 88 +++
.../src/main/proto/DatanodeClientProtocol.proto | 12 +
hadoop-hdds/rocks-native/pom.xml | 6 +-
.../ozone/client/MockXceiverClientFactory.java | 9 +-
.../client/checksum/FileChecksumBenchmark.java | 12 +
.../dist/dev-support/bin/dist-layout-stitching | 18 +
.../dist/src/main/compose/ozone/short-circuit.yaml | 25 +
.../src/main/compose/ozone/test-short-circuit.sh | 36 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +
.../src/main/smoketest/freon/read-write-key.robot | 7 +
.../smoketest/short-circuit/short-circuit.robot | 55 ++
.../dist/src/shell/ozone/ozone-functions.sh | 42 ++
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +
hadoop-ozone/integration-test/pom.xml | 11 +
.../hadoop/hdds/TestXceiverServerDomainSocket.java | 793 +++++++++++++++++++++
.../hadoop/hdds/scm/TestXceiverClientManager.java | 6 +-
.../hdds/scm/TestXceiverClientManagerSC.java | 109 +++
.../TestDNDataDistributionFinalization.java | 11 +-
.../TestScmDataDistributionFinalization.java | 8 +-
.../ozone/client/rpc/read/InputStreamTests.java | 22 +-
.../client/rpc/read/TestChunkInputStream.java | 16 +-
.../client/rpc/read/TestDomainSocketFactory.java | 81 +++
.../client/rpc/read/TestLocalChunkInputStream.java | 163 +++++
.../ozone/om/TestOmContainerLocationCache.java | 4 +
.../ozone/om/service/TestBlockDeletionService.java | 7 +-
pom.xml | 183 +++++
72 files changed, 4540 insertions(+), 144 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org