You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2020/06/06 11:51:36 UTC
[hadoop-ozone] 01/02: Revert "Revert "HDDS-3710. Merge archived
jacoco coverage results (#1014)""
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-3710-test
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 36738a0ea740f90d06dc07163f0b6ae06cc46e2d
Author: Elek Márton <el...@apache.org>
AuthorDate: Sat Jun 6 13:48:46 2020 +0200
Revert "Revert "HDDS-3710. Merge archived jacoco coverage results (#1014)""
This reverts commit 322bcdd2ce958450132896ad0d42cc97927b4fec.
---
.github/coverage-report.sh | 56 ++++++++++++++++++++++++++++++++
.github/workflows/post-commit.yml | 40 ++++++++++++++++++-----
hadoop-ozone/dev-support/checks/sonar.sh | 12 ++++++-
3 files changed, 99 insertions(+), 9 deletions(-)
diff --git a/.github/coverage-report.sh b/.github/coverage-report.sh
new file mode 100755
index 0000000..b345a87
--- /dev/null
+++ b/.github/coverage-report.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#This script can merge the downloaded jacoco files from other artifacts
+#Works only in github environment where the jacoco .exec files are downloaded
+#from job artifacts
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+cd "$DIR/.." || exit 1
+
+set -ex
+
+REPORT_DIR="$DIR/../target/coverage"
+
+mkdir -p "$REPORT_DIR"
+
+#Install jacoco cli
+mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.2:copy -Dartifact=org.jacoco:org.jacoco.cli:0.8.5:jar:nodeps
+
+jacoco() {
+ java -jar target/dependency/org.jacoco.cli-0.8.5-nodeps.jar "$@"
+}
+
+#Merge all the jacoco.exec files
+jacoco merge $(find target/artifacts -name jacoco-combined.exec) --destfile "$REPORT_DIR/jacoco-all.exec"
+
+rm -rf target/coverage-classes || true
+mkdir -p target/coverage-classes
+
+#Unzip all the classes from the last build
+find hadoop-ozone/dist/target/*/share/ozone/lib -name "hadoop-*.jar" | \
+ grep -E 'hadoop-ozone-|hadoop-hdds-' | \
+ grep -v -E 'legacy|current|tests' | \
+ xargs -n1 unzip -o -q -d target/coverage-classes
+
+#Exclude some classes from the coverage
+find target/coverage-classes -name proto -type d | xargs rm -rf
+find target/coverage-classes -name generated -type d | xargs rm -rf
+find target/coverage-classes -name v1 -type d | xargs rm -rf
+find target/coverage-classes -name freon -type d | xargs rm -rf
+find target/coverage-classes -name genesis -type d | xargs rm -rf
+
+#generate the reports
+jacoco report "$REPORT_DIR/jacoco-all.exec" --classfiles target/coverage-classes --html "$REPORT_DIR/all" --xml "$REPORT_DIR/all.xml"
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 5a920e5..fc2dda1 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -58,13 +58,6 @@ jobs:
- uses: ./.github/buildenv
with:
args: ./hadoop-ozone/dev-support/checks/unit.sh
- - uses: ./.github/buildenv
- if: github.repository == 'apache/hadoop-ozone'
- with:
- args: ./hadoop-ozone/dev-support/checks/sonar.sh
- env:
- SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/upload-artifact@master
if: always()
with:
@@ -226,4 +219,35 @@ jobs:
if: always()
with:
name: it-ozone
- path: mnt/ozone/target/integration
\ No newline at end of file
+ path: mnt/ozone/target/integration
+ coverage:
+ name: coverage
+ runs-on: ubuntu-18.04
+ needs:
+ - it-ozone
+ - it-hdds-om
+ - it-client
+ - it-filesystem-contract
+ - it-filesystem
+ - it-freon
+ - unit
+ steps:
+ - uses: actions/checkout@v2
+ - uses: ./.github/buildenv
+ with:
+ args: ./hadoop-ozone/dev-support/checks/build.sh
+ - uses: actions/download-artifact@v2
+ with:
+ path: target/artifacts
+ - run: ./.github/coverage-report.sh
+ - uses: ./.github/buildenv
+ if: github.repository == 'apache/hadoop-ozone'
+ with:
+ args: ./hadoop-ozone/dev-support/checks/sonar.sh
+ env:
+ SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - uses: actions/upload-artifact@master
+ with:
+ name: coverage
+ path: target/coverage
diff --git a/hadoop-ozone/dev-support/checks/sonar.sh b/hadoop-ozone/dev-support/checks/sonar.sh
index 55f219b..5ab3308 100755
--- a/hadoop-ozone/dev-support/checks/sonar.sh
+++ b/hadoop-ozone/dev-support/checks/sonar.sh
@@ -13,11 +13,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+set -x
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
-cd "$DIR/../../.." || exit 1
+PROJECT_DIR="$(realpath "$DIR"/../../..)"
+cd "$PROJECT_DIR" || exit 1
if [ ! "$SONAR_TOKEN" ]; then
echo "SONAR_TOKEN environment variable should be set"
exit 1
fi
+
+#Workaround: Sonar expects per-project Sonar XML report, but we have one, combined. Sonar seems to handle it well.
+# Only the classes from the current project will be used. We can copy the same, combined report to all the subprojects.
+if [ -f "$PROJECT_DIR/target/coverage/all.xml" ]; then
+ find "$PROJECT_DIR" -name pom.xml | grep -v target | xargs dirname | xargs -n1 -IDIR mkdir -p DIR/target/coverage/
+ find "$PROJECT_DIR" -name pom.xml | grep -v target | xargs dirname | xargs -n1 -IDIR cp "$PROJECT_DIR/target/coverage/all.xml" DIR/target/coverage/
+fi
+
mvn -B verify -DskipShade -DskipTests -Dskip.yarn org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.0.1398:sonar -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=hadoop-ozone
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org