You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by da...@apache.org on 2024/08/21 23:33:44 UTC
(kafka) branch trunk updated: MINOR: Github build enable Gradle cache and add checkstyle annotations (#16948)
This is an automated email from the ASF dual-hosted git repository.
davidarthur pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 95ef75aa48d MINOR: Github build enable Gradle cache and add checkstyle annotations (#16948)
95ef75aa48d is described below
commit 95ef75aa48db054540217f1ac319538803c82a07
Author: David Arthur <mu...@gmail.com>
AuthorDate: Wed Aug 21 19:33:37 2024 -0400
MINOR: Github build enable Gradle cache and add checkstyle annotations (#16948)
This patch allows the gradle cache to be used on GitHub Actions.
Also adds a Python script to parse checkstyle reports and produce GitHub annotations on PRs.
Reviewers: Chia-Ping Tsai <ch...@gmail.com>
---
.github/scripts/checkstyle.py | 98 +++++++++++++++++++++++++++++++++++++++++++
.github/workflows/pr.yml | 27 ++++++++++--
settings.gradle | 4 +-
3 files changed, 125 insertions(+), 4 deletions(-)
diff --git a/.github/scripts/checkstyle.py b/.github/scripts/checkstyle.py
new file mode 100644
index 00000000000..d6af54c438e
--- /dev/null
+++ b/.github/scripts/checkstyle.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from glob import glob
+import logging
+import os
+import os.path
+import sys
+from typing import Tuple, Optional
+import xml.etree.ElementTree
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stderr)
+handler.setLevel(logging.DEBUG)
+logger.addHandler(handler)
+
+
+def get_env(key: str) -> str:
+ value = os.getenv(key)
+ logger.debug(f"Read env {key}: {value}")
+ return value
+
+
+def parse_report(workspace_path, fp) -> Tuple[int, int]:
+ stack = []
+ errors = []
+ file_count = 0
+ error_count = 0
+ for (event, elem) in xml.etree.ElementTree.iterparse(fp, events=["start", "end"]):
+ if event == "start":
+ stack.append(elem)
+ if elem.tag == "file":
+ file_count += 1
+ errors.clear()
+ if elem.tag == "error":
+ logger.debug(f"Found checkstyle error: {elem.attrib}")
+ errors.append(elem)
+ error_count += 1
+ elif event == "end":
+ if elem.tag == "file" and len(errors) > 0:
+ filename = elem.get("name")
+ rel_path = os.path.relpath(filename, workspace_path)
+ logger.debug(f"Outputting errors for file: {elem.attrib}")
+ for error in errors:
+ line = error.get("line")
+ col = error.get("column")
+ severity = error.get("severity")
+ message = error.get('message')
+ title = f"Checkstyle {severity}"
+ print(f"::notice file={rel_path},line={line},col={col},title={title}::{message}")
+ stack.pop()
+ else:
+ logger.error(f"Unhandled xml event {event}: {elem}")
+ return file_count, error_count
+
+
+if __name__ == "__main__":
+ """
+ Parse checkstyle XML reports and generate GitHub annotations.
+
+ See: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-notice-message
+ """
+ if not os.getenv("GITHUB_WORKSPACE"):
+ print("This script is intended to by run by GitHub Actions.")
+ exit(1)
+
+ reports = glob(pathname="**/checkstyle/*.xml", recursive=True)
+ logger.debug(f"Found {len(reports)} checkstyle reports")
+ total_file_count = 0
+ total_error_count = 0
+
+ workspace_path = get_env("GITHUB_WORKSPACE") # e.g., /home/runner/work/apache/kafka
+
+ for report in reports:
+ with open(report, "r") as fp:
+ logger.debug(f"Parsing report file: {report}")
+ file_count, error_count = parse_report(workspace_path, fp)
+ if error_count == 1:
+ logger.debug(f"Checked {file_count} files from {report} and found 1 error")
+ else:
+ logger.debug(f"Checked {file_count} files from {report} and found {error_count} errors")
+ total_file_count += file_count
+ total_error_count += error_count
+ exit(0)
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 7f22a2fa5da..a7e0826091a 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -31,13 +31,16 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 8, 11, 17, 21 ]
+ java: [ 21, 17, 11, 8 ]
name: Compile and Check Java ${{ matrix.java }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
- name: Setup Gradle
uses: ./.github/actions/setup-gradle
with:
@@ -45,8 +48,26 @@ jobs:
gradle-cache-read-only: true
develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Compile and validate
- # Publish build scans for now. This will only work on PRs from apache/kafka, not public forks.
+ # Gradle flags
+ # --build-cache: Let Gradle restore the build cache
+ # --info: For now, we'll generate lots of logs while setting up the GH Actions
+ # --scan: Attempt to publish build scans in PRs. This will only work on PRs from apache/kafka, not public forks.
run: ./gradlew --build-cache --info --scan check -x test
+ - name: Archive check reports
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: check-reports-${{ matrix.java }}
+ path: |
+ **/build/**/*.html
+ if-no-files-found: ignore
+ - name: Annotate checkstyle errors
+ # Avoid duplicate annotations, only run on java 21
+ # For now, limit to "gh-" PRs
+ if: ${{ contains(github.head_ref, 'gh-') && failure() && matrix.java == '21' }}
+ run: python .github/scripts/checkstyle.py
+ env:
+ GITHUB_WORKSPACE: ${{ github.workspace }}
test:
needs: validate
@@ -54,7 +75,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11, 17 ]
+ java: [ 17, 11 ]
if: contains(github.head_ref, 'gh-') # For now, only run tests on "gh-*" PRs
name: JUnit tests Java ${{ matrix.java }}
steps:
diff --git a/settings.gradle b/settings.gradle
index 662864b2224..ed85252d068 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -45,7 +45,9 @@ develocity {
buildCache {
local {
- enabled = !isCI
+ // This allows the build cache to be used locally or on GitHub Actions.
+ // Using the cache on GH should be safe since each job is run on a new VM
+ enabled = !isJenkins
}
remote(develocity.buildCache) {