You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2020/06/13 15:11:25 UTC
[hadoop-ozone] branch HDDS-3757 updated (07b42fe -> 4b1cc4b)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch HDDS-3757
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from 07b42fe retrigger ci
add b982b64 server based coverage collection
add 9798cdb HDDS-3686. Insert an argument of ozone shell to accept jvm arguments (#991)
add 38e7ca0 HDDS-2879. Update return description of OzoneManager#openKey(OmKeyArgs args) (#898)
add 2ff6c89 HDDS-3682. Recon UI: Add interactive visualization for file size counts (#1032)
add ef24b11 HDDS-3622. Implement rocksdb tool to parse scm db (#945)
add b9b621d HDDS-3715. Improvement for OzoneFS client to work with Hadoop 2.7.3. (#1036)
add 4da05e4 HDDS-3750. Improve SCM performance with 3.2% by avoid stream.collect (#1035)
add 1238769 HDDS-3760. Avoid UUID#toString call in Pipeline#getProtobufMessage (#1043)
add f7e95d9 HDDS-3726. Upload code coverage to Codecov and enable checks in PR workflow of Github Actions (#1047)
add 33992bd HDDS-3749. Improve OM performance with 3.7% by avoid stream.collect (#1034)
add 53395a0 HDDS-3479. Fix checkstyle issue. (#1052)
add f7fcadc HDDS-3612. Document details of bucket mount design (#1009)
add cee76a8 HDDS-3481. SCM ask too many datanodes to replicate the same container. (#1048)
add 78255dd HDDS-3476. Use persisted transaction info during OM startup in OM StateMachine. (#986)
add 6cbe8d0 HDDS-3753. SCMPipelineManager's createPipeline method has redundant synchronization modifiers. (#1039)
add bd88eec HDDS-3777. Reduce recon UI build time (#1065)
add 3876e04 Merge remote-tracking branch 'origin/master' into HDDS-3757
add 9378b0f test acceptance only
add b833fef remove dependency
new ac3811b Revert "remove dependency"
new 4b1cc4b Revert "test acceptance only"
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 27 +-
.github/workflows/pr.yml | 50 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 5 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 28 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 -
.../common/src/main/resources/ozone-default.xml | 9 -
.../docs/content/design/ozone-volume-management.md | 31 +-
.../apache/hadoop/hdds/utils/db/DBCheckpoint.java | 29 -
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 22 -
.../hdds/scm/container/ReplicationManager.java | 18 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 72 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 2 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 8 +-
.../hdds/scm/server/StorageContainerManager.java | 3 +-
.../hdds/scm/container/TestReplicationManager.java | 13 +-
hadoop-hdds/test-utils/pom.xml | 6 +
.../java/org/apache/hadoop/test/JacocoServer.java | 111 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 12 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 19 +-
.../ozone/om/protocol/OzoneManagerHAProtocol.java | 39 -
.../om/protocol/OzoneManagerServerProtocol.java | 28 -
hadoop-ozone/dev-support/checks/acceptance.sh | 2 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 4 +-
hadoop-ozone/dev-support/checks/sonar.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
hadoop-ozone/dist/pom.xml | 17 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 19 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 4 -
.../dist/src/shell/hdds/hadoop-functions.sh | 6 +
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 4 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 18 -
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 9 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 4 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 26 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 28 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 28 -
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 20 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 108 +-
.../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 6 +-
.../hadoop/ozone/om/ratis/OMRatisSnapshotInfo.java | 116 +-
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 13 +
.../ozone/om/ratis/OzoneManagerRatisServer.java | 21 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 45 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 67 +
.../ozone/om/request/key/OMKeyCommitRequest.java | 12 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 34 +-
.../ozone/om/ratis/TestOMRatisSnapshotInfo.java | 65 -
.../om/ratis/TestOzoneManagerRatisServer.java | 22 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 20 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 122 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 2 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 7 +-
hadoop-ozone/pom.xml | 1 +
hadoop-ozone/recon/pom.xml | 25 +-
.../webapps/recon/ozone-recon-web/LICENSE | 3335 +++-
.../webapps/recon/ozone-recon-web/README.md | 13 +-
.../webapps/recon/ozone-recon-web/api/db.json | 64 +-
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../webapps/recon/ozone-recon-web/package.json | 8 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 17167 +++++++++++++++++++
.../src/components/multiSelect/multiSelect.tsx | 123 +
.../src/components/navBar/navBar.tsx | 5 +
.../src/constants/breadcrumbs.constants.tsx | 3 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 5 +
.../src/views/datanodes/datanodes.tsx | 22 +-
.../overview.less => insights/insights.less} | 21 +-
.../src/views/insights/insights.tsx | 279 +
.../views/missingContainers/missingContainers.tsx | 10 +-
.../src/views/overview/overview.tsx | 8 +-
.../src/views/pipelines/pipelines.tsx | 9 +-
.../webapps/recon/ozone-recon-web/yarn.lock | 13625 ---------------
.../org/apache/hadoop/ozone/debug/DBScanner.java | 145 +
.../ListTables.java} | 34 +-
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 3 +-
.../RDBParser.java} | 34 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
pom.xml | 2 +-
87 files changed, 21774 insertions(+), 14617 deletions(-)
create mode 100644 hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/JacocoServer.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerServerProtocol.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOMRatisSnapshotInfo.java
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/pnpm-lock.yaml
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/multiSelect/multiSelect.tsx
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{overview/overview.less => insights/insights.less} (76%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/insights/insights.tsx
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{shell/token/PrintTokenHandler.java => debug/ListTables.java} (59%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{shell/bucket/GetAclBucketHandler.java => debug/RDBParser.java} (59%)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 02/02: Revert "test acceptance only"
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-3757
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 4b1cc4bda453db33d1fdf05e84ec78bbf4dd134a
Author: Elek Márton <el...@apache.org>
AuthorDate: Sat Jun 13 17:10:18 2020 +0200
Revert "test acceptance only"
This reverts commit 9378b0f9839c42fa8f6c65a7672ac7e8714957e2.
---
.github/workflows/post-commit.yml | 227 ++++++++++++++++++++++++++++++++++++++
1 file changed, 227 insertions(+)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index aed008f..807e2b2 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -16,6 +16,87 @@ name: build-branch
on:
- push
jobs:
+ build:
+ name: compile
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ - uses: actions/cache@v2
+ with:
+ path: |
+ ~/.pnpm-store
+ **/node_modules
+ key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+ restore-keys: |
+ ${{ runner.os }}-pnpm-
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/build.sh
+ rat:
+ name: rat
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/rat.sh
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: rat
+ path: target/rat
+ author:
+ name: author
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/author.sh
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: author
+ path: target/author
+ unit:
+ name: unit
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/unit.sh
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: unit
+ path: target/unit
+ checkstyle:
+ name: checkstyle
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/checkstyle.sh
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: checkstyle
+ path: target/checkstyle
+ findbugs:
+ name: findbugs
+ runs-on: ubuntu-18.04
+ steps:
+ - uses: actions/checkout@master
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/findbugs.sh
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: findbugs
+ path: target/findbugs
acceptance:
name: acceptance
runs-on: ubuntu-18.04
@@ -62,3 +143,149 @@ jobs:
rm -rf ~/.m2/repository/org/apache/hadoop/hdds
rm -rf ~/.m2/repository/org/apache/hadoop/ozone
if: always()
+ it-freon:
+ name: it-freon
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfreon
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-freon
+ path: mnt/ozone/target/integration
+ it-filesystem:
+ name: it-filesystem
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-filesystem
+ path: mnt/ozone/target/integration
+ it-filesystem-contract:
+ name: it-filesystem-contract
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem-contract
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-filesystem-contract
+ path: mnt/ozone/target/integration
+ it-client:
+ name: it-client
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pclient
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-client
+ path: mnt/ozone/target/integration
+ it-hdds-om:
+ name: it-hdds-om
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Phdds-om
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-hdds-om
+ path: mnt/ozone/target/integration
+ it-ozone:
+ name: it-ozone
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pozone
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-ozone
+ path: mnt/ozone/target/integration
+ coverage:
+ name: coverage
+ runs-on: ubuntu-18.04
+ needs:
+ - it-ozone
+ - it-hdds-om
+ - it-client
+ - it-filesystem-contract
+ - it-filesystem
+ - it-freon
+ - acceptance
+ - unit
+ steps:
+ - uses: actions/checkout@v2
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/build.sh
+ - uses: actions/download-artifact@v2
+ with:
+ path: target/artifacts
+ - run: ./.github/coverage-report.sh
+ - uses: ./.github/buildenv
+ if: github.repository == 'apache/hadoop-ozone'
+ with:
+ args: ./hadoop-ozone/dev-support/checks/sonar.sh
+ env:
+ SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - name: Upload coverage to Codecov
+ uses: codecov/codecov-action@v1
+ with:
+ file: ./target/coverage/all.xml
+ name: codecov-umbrella
+ fail_ci_if_error: true
+ - uses: actions/upload-artifact@master
+ with:
+ name: coverage
+ path: target/coverage
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/02: Revert "remove dependency"
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-3757
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit ac3811b6cc216c6e0ab93c32a22e1963a2f881a9
Author: Elek Márton <el...@apache.org>
AuthorDate: Sat Jun 13 17:09:06 2020 +0200
Revert "remove dependency"
This reverts commit b833fef49c0f8aa99f1221cf79a58d9b9da9940d.
---
.github/workflows/post-commit.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 31372eb..aed008f 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -19,6 +19,8 @@ jobs:
acceptance:
name: acceptance
runs-on: ubuntu-18.04
+ needs:
+ - build
steps:
- uses: actions/cache@v2
with:
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org