You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ra...@apache.org on 2026/03/28 10:38:01 UTC
(storm) branch master updated: Add script and workflow to auto-update license files (#8439)
This is an automated email from the ASF dual-hosted git repository.
rabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/storm.git
The following commit(s) were added to refs/heads/master by this push:
new cfb9c0057 Add script and workflow to auto-update license files (#8439)
cfb9c0057 is described below
commit cfb9c0057c3af5e65ab9b44d56ee18b84f720abb
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Sat Mar 28 11:37:56 2026 +0100
Add script and workflow to auto-update license files (#8439)
- Add dev-tools/update-license-files.py: standalone script that regenerates
DEPENDENCY-LICENSES and the binary deps section of LICENSE-binary
- Add .github/workflows/update-license-files.yml: runs on push to master
and auto-creates a PR via peter-evans/create-pull-request when license
files drift after dependency changes
- Remove Check-Updated-License-Files from CI matrix since the auto-update
workflow replaces manual license validation
---
.github/workflows/maven.yaml | 2 +-
.../{maven.yaml => update-license-files.yml} | 76 +++--
dev-tools/update-license-files.py | 338 +++++++++++++++++++++
3 files changed, 384 insertions(+), 32 deletions(-)
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
index efff6c3c8..42603b8c0 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven.yaml
@@ -29,7 +29,7 @@ jobs:
matrix:
os: [ ubuntu-latest ]
java: [ 17, 21 ]
- module: [ Client, Server, Core, External, Check-Updated-License-Files, Integration-Test ]
+ module: [ Client, Server, Core, External, Integration-Test ]
experimental: [false]
fail-fast: false
steps:
diff --git a/.github/workflows/maven.yaml b/.github/workflows/update-license-files.yml
similarity index 51%
copy from .github/workflows/maven.yaml
copy to .github/workflows/update-license-files.yml
index efff6c3c8..518c6751c 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/update-license-files.yml
@@ -13,56 +13,70 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Java CI with Maven
+name: Update License Files
on:
- pull_request:
- branches: [ "master" ]
push:
branches: [ "master" ]
+ workflow_dispatch:
+
+permissions:
+ contents: write
+ pull-requests: write
jobs:
- build:
- runs-on: ${{ matrix.os }}
- continue-on-error: ${{ matrix.experimental }}
- strategy:
- matrix:
- os: [ ubuntu-latest ]
- java: [ 17, 21 ]
- module: [ Client, Server, Core, External, Check-Updated-License-Files, Integration-Test ]
- experimental: [false]
- fail-fast: false
+ update-license-files:
+ runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
- uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
+
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.10'
- - name: Set up Node
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
- with:
- node-version: 20
- - name: Set up Ruby
- uses: ruby/setup-ruby@4dc28cf14d77b0afa6832d9765ac422dbf0dfedd # v1.298.0
- with:
- ruby-version: '2.7'
- - name: Set up JDK ${{ matrix.java }}
+
+ - name: Set up JDK 17
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
distribution: temurin
- java-version: ${{ matrix.java }}
- - name: Ensure a clean state without storm artifacts
+ java-version: 17
+
+ - name: Ensure clean state without storm artifacts
run: rm -rf ~/.m2/repository/org/apache/storm
- - name: Set up project dependencies
- run: /bin/bash ./dev-tools/gitact/gitact-install.sh `pwd`
- - name: Run build
- run: |
- export JDK_VERSION=${{ matrix.java }}
- export USER=github
- /bin/bash ./dev-tools/gitact/gitact-script.sh `pwd` ${{ matrix.module }};
+
+ - name: Build Storm
+ run: mvn clean install -B -DskipTests -Dcheckstyle.skip -Dpmd.skip
+ env:
+ MAVEN_OPTS: -Xmx2048m
+
+ - name: Update license files
+ run: python3 dev-tools/update-license-files.py --skip-build-storm
+
+ - name: Create Pull Request
+ uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ commit-message: "Regenerate license files after dependency changes"
+ committer: GitHub <no...@github.com>
+ author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+ signoff: true
+ branch: update-license-files
+ delete-branch: true
+ title: "[Automated] Update license files after dependency changes"
+ body: |
+ This PR was automatically generated after a push to master caused
+ license files to become out of date.
+
+ Updated files:
+ - `DEPENDENCY-LICENSES`
+ - `LICENSE-binary` (binary dependencies section)
+
+ Please review the changes and merge if correct.
+ labels: dependencies
diff --git a/dev-tools/update-license-files.py b/dev-tools/update-license-files.py
new file mode 100755
index 000000000..b46cc248c
--- /dev/null
+++ b/dev-tools/update-license-files.py
@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Usage: update-license-files.py [--skip-build-storm]
+
+Regenerates DEPENDENCY-LICENSES and the binary dependencies section of LICENSE-binary.
+Run this after changing dependencies to bring license files up to date.
+
+Depends on "requests" is NOT required. Only needs Maven and Python 3.
+"""
+
+from contextlib import contextmanager
+from pathlib import Path
+import os
+import subprocess
+import shlex
+import shutil
+import filecmp
+import re
+import argparse
+
+project_root = Path(__file__).resolve().parent.parent
+update_dependency_licenses_cmd = ('mvn license:aggregate-add-third-party@generate-and-check-licenses -Dlicense.skipAggregateAddThirdParty=false -B')
+
+LICENSE_BINARY_SEPARATOR = '----------------------------END OF SOURCE NOTICES -------------------------------------------'
+
+LICENSE_BINARY_PREAMBLE = """
+
+The following dependencies are included in the binary Storm distributions, in addition to the source dependencies listed above.
+The license texts of these dependencies can be found in the licenses directory.
+"""
+
+
+@contextmanager
+def cd(newdir):
+ prevdir = Path.cwd()
+ os.chdir(newdir.expanduser())
+ try:
+ yield
+ finally:
+ os.chdir(prevdir)
+
+
+def read_lines(path):
+ with open(path) as f:
+ return f.readlines()
+
+
+def build_storm():
+ print("Building Storm")
+ subprocess.check_call(shlex.split(
+ 'mvn clean install -B -DskipTests -Dcheckstyle.skip -Dpmd.skip'
+ ))
+ print("Done building Storm")
+
+
+def generate_dependency_licenses():
+ """Generates DEPENDENCY-LICENSES in target/. The committed DEPENDENCY-LICENSES is not modified."""
+ print('Generating DEPENDENCY-LICENSES')
+ cmd = (update_dependency_licenses_cmd +
+ ' -Dlicense.thirdPartyFilename=DEPENDENCY-LICENSES' +
+ ' -Dlicense.outputDirectory=target')
+ subprocess.check_call(shlex.split(cmd))
+ print('Done generating DEPENDENCY-LICENSES')
+
+
+def generate_storm_dist_license_report():
+ with cd(project_root / 'storm-dist' / 'binary'):
+ print('')
+ print('Generating storm-dist license report')
+ subprocess.check_call(shlex.split(update_dependency_licenses_cmd))
+ print('Done generating storm-dist license report')
+
+
+def extract_license_report_maven_coordinates(lines):
+ """Extract Maven coordinates from license report lines.
+ Lines like: ' * Checker Qual (org.checkerframework:checker-qual:2.5.2 - https://checkerframework.org)'
+ """
+ matches = map(lambda line: re.match(
+ r'\s+\*.*\((?P<gav>.*) \- .*\).*', line), lines)
+ return set(map(lambda match: match.group('gav'), filter(lambda match: match is not None, matches)))
+
+
+def extract_dependency_list_maven_coordinates(lines):
+ """Extract Maven coordinates from 'mvn dependency:list' output.
+ Lines like: ' com.google.code.findbugs:jsr305:jar:3.0.2 -- module jsr305 (auto)'
+ """
+ matches = map(lambda line: re.match(
+ r'\s+(?P<group>\S*)\:(?P<artifact>\S*)\:(?P<type>\S*)\:(?P<version>\S*)', line), lines)
+ return set(map(lambda match: match.group('group') + ':' + match.group('artifact') + ':' + match.group('version'),
+ filter(lambda match: match is not None, matches)))
+
+
+def get_shaded_dep_coordinates():
+ """Gets the set of Maven coordinates for storm-shaded-deps (excluding Storm's own modules)."""
+ with cd(project_root / 'storm-shaded-deps'):
+ print("Generating dependency list for storm-shaded-deps")
+ subprocess.check_call(shlex.split(
+ 'mvn dependency:list -DoutputFile=target/deps-list -Dmdep.outputScope=false -DincludeScope=compile -B'))
+ print("Done generating dependency list for storm-shaded-deps")
+ shaded_dep_coordinates = extract_dependency_list_maven_coordinates(
+ read_lines(project_root / 'storm-shaded-deps' / 'target' / 'deps-list'))
+ shaded_dep_coordinates = set(filter(lambda coordinate: 'org.apache.storm:' not in coordinate, shaded_dep_coordinates))
+ print('storm-shaded-deps dependencies: ' + str(shaded_dep_coordinates))
+ print('')
+ return shaded_dep_coordinates
+
+
+def parse_grouped_license_file(lines):
+ """Parse a license report file (DEPENDENCY-LICENSES format) into structured groups.
+
+ Returns a list of (header_line, [entry_lines]) tuples, preserving blank lines and formatting.
+ The header_line is the license group name (e.g. ' Apache License, Version 2.0\\n').
+ entry_lines are the dependency lines under that group (e.g. ' * Foo (g:a:v - url)\\n').
+ """
+ groups = []
+ current_header = None
+ current_entries = []
+
+ for line in lines:
+ # Skip the file header (first few lines before any license group)
+ stripped = line.rstrip('\n')
+
+ # License group headers are indented with 4 spaces and have non-whitespace content
+ if re.match(r' \S', line) and not line.strip().startswith('*'):
+ if current_header is not None:
+ groups.append((current_header, current_entries))
+ current_header = line
+ current_entries = []
+ elif current_header is not None:
+ current_entries.append(line)
+
+ if current_header is not None:
+ groups.append((current_header, current_entries))
+
+ return groups
+
+
+def filter_groups_to_coordinates(groups, target_coordinates):
+ """Filter license groups to only include entries whose coordinates are in the target set.
+
+ Returns lines for the filtered license report section.
+ """
+ result_lines = []
+ gav_pattern = re.compile(r'\s+\*.*\((?P<gav>.*) \- .*\).*')
+
+ for header, entries in groups:
+ filtered_entries = []
+ for entry in entries:
+ match = gav_pattern.match(entry)
+ if match:
+ if match.group('gav') in target_coordinates:
+ filtered_entries.append(entry)
+ # Keep blank lines between entries only if we have entries
+ if filtered_entries:
+ result_lines.append('\n')
+ result_lines.append(header)
+ result_lines.append('\n')
+ for entry in filtered_entries:
+ result_lines.append(entry)
+
+ return result_lines
+
+
+def update_dependency_licenses():
+ """Copy target/DEPENDENCY-LICENSES to root DEPENDENCY-LICENSES.
+ Returns True if the file changed."""
+ src = project_root / 'target' / 'DEPENDENCY-LICENSES'
+ dst = project_root / 'DEPENDENCY-LICENSES'
+ if dst.exists() and filecmp.cmp(src, dst, shallow=False):
+ print('DEPENDENCY-LICENSES is already up to date')
+ return False
+ shutil.copy2(src, dst)
+ print('Updated DEPENDENCY-LICENSES')
+ return True
+
+
+def merge_groups(base_groups, extra_groups, extra_coordinates):
+ """Merge extra_groups entries into base_groups for coordinates in extra_coordinates
+ that are not already present in base_groups.
+
+ Returns a new list of (header, [entry_lines]) tuples.
+ """
+ # Collect all coordinates already in base_groups
+ gav_pattern = re.compile(r'\s+\*.*\((?P<gav>.*) \- .*\).*')
+ base_coords = set()
+ for _, entries in base_groups:
+ for entry in entries:
+ match = gav_pattern.match(entry)
+ if match:
+ base_coords.add(match.group('gav'))
+
+ # Find extra entries to add (in extra_coordinates but not in base)
+ missing_coords = extra_coordinates - base_coords
+ if not missing_coords:
+ return base_groups
+
+ print(f'Adding {len(missing_coords)} shaded-deps entries from DEPENDENCY-LICENSES to LICENSE-binary')
+
+ # Build a map: header_text -> list of entry lines to add
+ extra_by_header = {}
+ for header, entries in extra_groups:
+ header_key = header.strip()
+ for entry in entries:
+ match = gav_pattern.match(entry)
+ if match and match.group('gav') in missing_coords:
+ extra_by_header.setdefault(header_key, []).append(entry)
+
+ # Merge into base_groups
+ result = []
+ seen_headers = set()
+ for header, entries in base_groups:
+ header_key = header.strip()
+ seen_headers.add(header_key)
+ merged_entries = list(entries)
+ if header_key in extra_by_header:
+ merged_entries.extend(extra_by_header[header_key])
+ result.append((header, merged_entries))
+
+ # Add any license groups that only exist in extra
+ for header_key, extra_entries in extra_by_header.items():
+ if header_key not in seen_headers:
+ result.append((' ' + header_key + '\n', extra_entries))
+
+ return result
+
+
+def groups_to_lines(groups):
+ """Convert parsed groups back to lines for writing."""
+ result_lines = []
+ for header, entries in sorted(groups, key=lambda g: g[0].strip().lower()):
+ result_lines.append('\n')
+ result_lines.append(header)
+ result_lines.append('\n')
+ for entry in entries:
+ # Skip blank lines that were part of the original inter-group spacing
+ if entry.strip():
+ result_lines.append(entry)
+ return result_lines
+
+
+def update_license_binary(shaded_dep_coordinates):
+ """Replace the binary dependencies section of LICENSE-binary.
+
+ Uses storm-dist/binary THIRD-PARTY.txt as the base (already has all binary deps grouped),
+ then merges in any storm-shaded-deps entries from DEPENDENCY-LICENSES that aren't already
+ covered by the binary THIRD-PARTY.txt.
+
+ Returns True if the file changed."""
+ license_binary_path = project_root / 'LICENSE-binary'
+ lines = read_lines(license_binary_path)
+
+ # Find the separator line
+ separator_idx = None
+ for i, line in enumerate(lines):
+ if LICENSE_BINARY_SEPARATOR in line:
+ separator_idx = i
+ break
+
+ if separator_idx is None:
+ print(f'ERROR: Could not find separator line in LICENSE-binary: "{LICENSE_BINARY_SEPARATOR}"')
+ return False
+
+ # Keep the static header (up to and including the separator)
+ static_part = lines[:separator_idx + 1]
+
+ # Use the binary THIRD-PARTY.txt as the base — it already contains all
+ # storm-dist/binary dependencies in the correct grouped format
+ binary_third_party_path = (project_root / 'storm-dist' / 'binary' / 'target' /
+ 'generated-sources' / 'license' / 'THIRD-PARTY.txt')
+ binary_groups = parse_grouped_license_file(read_lines(binary_third_party_path))
+
+ # Merge in storm-shaded-deps entries from DEPENDENCY-LICENSES
+ dep_licenses_lines = read_lines(project_root / 'target' / 'DEPENDENCY-LICENSES')
+ dep_licenses_groups = parse_grouped_license_file(dep_licenses_lines)
+ merged_groups = merge_groups(binary_groups, dep_licenses_groups, shaded_dep_coordinates)
+
+ binary_section = groups_to_lines(merged_groups)
+
+ # Compose the new file
+ preamble_lines = LICENSE_BINARY_PREAMBLE.splitlines(keepends=True)
+ new_content = static_part + preamble_lines + binary_section
+
+ # Compare with current content
+ if new_content == lines:
+ print('LICENSE-binary is already up to date')
+ return False
+
+ with open(license_binary_path, 'w') as f:
+ f.writelines(new_content)
+ print('Updated LICENSE-binary')
+ return True
+
+
+if __name__ == '__main__':
+ with cd(project_root):
+ parser = argparse.ArgumentParser(
+ description='Update Storm license files (DEPENDENCY-LICENSES and LICENSE-binary)')
+ parser.add_argument('--skip-build-storm', action='store_true',
+ help='skip building Storm (use if already built)')
+ args = parser.parse_args()
+
+ try:
+ if not args.skip_build_storm:
+ build_storm()
+ generate_dependency_licenses()
+ generate_storm_dist_license_report()
+
+ shaded_dep_coordinates = get_shaded_dep_coordinates()
+
+ dep_changed = update_dependency_licenses()
+ lic_changed = update_license_binary(shaded_dep_coordinates)
+
+ if dep_changed or lic_changed:
+ print('\nLicense files were updated. Please review the changes.')
+ else:
+ print('\nLicense files are already up to date. No changes made.')
+ except subprocess.CalledProcessError as e:
+ print(f'Command failed: {e}', flush=True)
+ exit(1)
+ except Exception as e:
+ print(f'Error updating license files: {e}', flush=True)
+ exit(1)