You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/07/08 18:30:44 UTC
(tooling-agents) branch main updated: Restructure reports into channels/ and pmcs/ directories
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-agents.git
The following commit(s) were added to refs/heads/main by this push:
new d267383 Restructure reports into channels/ and pmcs/ directories
d267383 is described below
commit d267383d841a61ea41ab11b75ba5018bf6701801
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Wed Jul 8 11:22:56 2026 -0700
Restructure reports into channels/ and pmcs/ directories
Directory layout:
- Channel reports move from flat gha_channel_{name}.md to channels/{name}.md
- Per-PMC security pages generated at pmcs/{pmc}.md (88 projects)
- channels/README.md and pmcs/README.md as directory indexes
- Drop gha_ prefix from all output filenames
- Drop atr_catalog.json
Security agent (major rewrite):
- Returns multi-file JSON: security.md + pmc-{name}.md + pmcs-index.md
- Main security.md is severity-grouped summary linking to PMC pages
- PMC pages have findings grouped by severity with workflow links
including #L{line} line numbers
- Progress tracking via ci-security-history namespace: diffs finding
hashes across runs, resolved findings noted on PMC pages
- Projects at zero findings get all-clear page with resolved history
- missing_codeowners removed from report output (kept in scan data)
Publishing detail:
- Fix (none) category display: shows trigger type instead
- Channel table split into Production and Staging sections
- Channel names link to channels/{name}.md
- Generates channels-index.md (routed to channels/README.md)
- Publishing risks: findings link to GitHub workflow URLs and
publishing_detail anchors
- npm_staging channel detection added (--staging flag, npm.pkg.github.com,
verdaccio)
Artifact verify:
- Wildcard package name matching across all channels: 5-7 candidate
names per channel using ASF naming patterns (apache-*, PMC fallback,
underscore/PascalCase variants)
- atr_catalog generation dropped
Brief:
- Links updated for new structure
- missing_codeowners removed from systemic issues
- Full Analysis references channels/ and pmcs/ directories
Review:
- Links updated for new structure
Orchestrator:
- Routes channel-* to channels/, pmc-* to pmcs/, indexes to */README.md
- Security handled via push_multi_file_output (was single file)
- Generates and pushes README.md as report landing page
---
gha-review/agents/gha_artifact_verify.py | 70 ++++--
gha-review/agents/gha_brief.py | 14 +-
gha-review/agents/gha_orchestrator.py | 214 +++++++++++------
gha-review/agents/gha_publishing_detail.py | 91 ++++++--
gha-review/agents/gha_security.py | 353 +++++++++++++++++++----------
5 files changed, 511 insertions(+), 231 deletions(-)
diff --git a/gha-review/agents/gha_artifact_verify.py b/gha-review/agents/gha_artifact_verify.py
index 742d43f..961a9e5 100644
--- a/gha-review/agents/gha_artifact_verify.py
+++ b/gha-review/agents/gha_artifact_verify.py
@@ -162,7 +162,62 @@ async def run(input_dict, tools):
# ── Build verification tasks ──
import asyncio
- verify_tasks = [] # (repo_data, channel, api_def, candidates_or_name)
+
+ def extract_pmc(name):
+ if name.startswith('incubator-'):
+ name = name[len('incubator-'):]
+ return name.split('-')[0]
+
+ def build_candidates(repo_name, channel):
+ base = repo_name
+ pmc = extract_pmc(base)
+ candidates = []
+
+ if channel == 'pypi':
+ candidates = [
+ f"apache-{base}", base, base.replace('-', '_'),
+ f"apache_{base}", base.replace('-', '.'),
+ ]
+ if pmc != base:
+ candidates.extend([f"apache-{pmc}", pmc])
+
+ elif channel == 'npm':
+ candidates = [
+ f"@apache/{base}", base, f"apache-{base}",
+ f"@apache-{pmc}/{base}",
+ ]
+ if pmc != base:
+ candidates.extend([f"@apache/{pmc}", pmc])
+
+ elif channel == 'crates_io':
+ candidates = [base, base.replace('-', '_'), f"apache-{base}"]
+ if pmc != base:
+ candidates.extend([pmc, f"apache-{pmc}"])
+
+ elif channel == 'docker_hub':
+ candidates = [f"apache/{base}", f"apache/{pmc}"]
+ if pmc != base:
+ candidates.append(f"apache/{base.replace(f'{pmc}-', '')}")
+
+ elif channel == 'rubygems':
+ candidates = [base, f"apache-{base}"]
+ if pmc != base:
+ candidates.append(pmc)
+
+ elif channel == 'nuget':
+ pascal = ''.join(w.capitalize() for w in base.split('-'))
+ candidates = [base, f"Apache.{base}", f"Apache.{pmc}",
+ pascal, f"Apache.{pascal}"]
+
+ else:
+ candidates = [base, f"apache-{base}"]
+ if pmc != base:
+ candidates.append(pmc)
+
+ seen = set()
+ return [c for c in candidates if c and c not in seen and not seen.add(c)]
+
+ verify_tasks = []
for repo_data in enriched_repos:
repo_full = repo_data['repo']
@@ -199,18 +254,8 @@ async def run(input_dict, tools):
if package_name:
candidates = [package_name]
- elif channel == 'pypi':
- candidates = [f"apache-{repo_name}", repo_name,
- repo_name.replace('-', '_'), f"apache_{repo_name}"]
- elif channel == 'npm':
- candidates = [f"@apache-{repo_name}/{repo_name}", repo_name,
- f"apache-{repo_name}"]
- elif channel == 'crates_io':
- candidates = [repo_name, repo_name.replace('-', '_')]
- elif channel == 'docker_hub':
- candidates = [f"apache/{repo_name}"]
else:
- candidates = [repo_name]
+ candidates = build_candidates(repo_name, channel)
verify_tasks.append((repo_data, channel, api, candidates))
@@ -340,7 +385,6 @@ async def run(input_dict, tools):
# ── Generate reports ──
files = {}
files['artifact-verification.md'] = gen_verification_report()
- files['atr-catalog.json'] = json.dumps(gen_atr_catalog(), indent=2)
# Store (one bulk write)
verify_ns = data_store.use_namespace(f"ci-artifact-verify:{github_owner}")
diff --git a/gha-review/agents/gha_brief.py b/gha-review/agents/gha_brief.py
index f5e9c0f..226be3e 100644
--- a/gha-review/agents/gha_brief.py
+++ b/gha-review/agents/gha_brief.py
@@ -137,7 +137,7 @@ async def run(input_dict, tools):
PUB = "publishing.md"
SEC = "security.md"
REVIEW = "review.md"
- JSON = "json-export.json"
+ JSON = "json_export.json"
def plural(n, singular, plural_form=None):
if plural_form is None:
@@ -227,13 +227,6 @@ async def run(input_dict, tools):
f"would execute in every affected workflow. "
f"([finding details]({SEC}#medium-findings))\n")
- codeowners_missing = check_counts.get("missing_codeowners", 0)
- if codeowners_missing:
- lines.append(f"**No workflow review gates.** {codeowners_missing} {plural(codeowners_missing, 'repo')} "
- f"have no CODEOWNERS file. Any committer can modify workflow files — adding "
- f"triggers, weakening permissions, or introducing injection patterns — "
- f"without mandatory security review.\n")
-
# --- Section 4: What to do ---
lines.append("## Recommended Actions\n")
@@ -277,6 +270,11 @@ async def run(input_dict, tools):
lines.append(f"- [{REVIEW}]({REVIEW}) — combined risk assessment with attack scenarios")
lines.append(f"- [{PUB}]({PUB}) — which repos publish what, where, and how")
lines.append(f"- [{SEC}]({SEC}) — all {total_findings} security findings by repo")
+ lines.append(f"- [publishing_detail.md](publishing_detail.md) — enriched analysis: secrets, action SHAs, triggers, run history")
+ lines.append(f"- [publishing_risks.md](publishing_risks.md) — dangerous publishing patterns")
+ lines.append(f"- [artifact_verification.md](artifact_verification.md) — registry cross-reference")
+ lines.append(f"- [channels/](channels/) — per-channel publishing detail (PyPI, npm, Maven, Docker Hub, etc.)")
+ lines.append(f"- [pmcs/](pmcs/) — per-project security reports with finding history")
lines.append(f"- [{JSON}]({JSON}) — machine-readable data ([query examples](README.md#jq-examples))")
lines.append(f"- [README.md](README.md) — report guide, JSON schema, jq commands")
lines.append("")
diff --git a/gha-review/agents/gha_orchestrator.py b/gha-review/agents/gha_orchestrator.py
index 0dbd6ae..709f0f4 100644
--- a/gha-review/agents/gha_orchestrator.py
+++ b/gha-review/agents/gha_orchestrator.py
@@ -2,42 +2,28 @@
gha_orchestrator
Runs the full CI/CD security + publishing analysis pipeline for a GitHub org.
-Single pass, full reports, pushes to one private repo.
+Single pass, full reports, pushes to one repo.
Inputs:
github_owner — GitHub org to scan (e.g. "apache")
- read_pat — PAT with read access to org repos (for prefetch + run history)
- write_repo — repo to push reports to (e.g. "apache/tooling-agents-private")
- write_directory — directory within repo (e.g. "gha-review")
+ read_pat — PAT with read access to org repos
+ write_repo — repo to push reports to
+ write_directory — directory within repo (e.g. "gha-review/reports")
write_pat — PAT with write access to write_repo
- fresh — "true" to clear all cached data and start from scratch (default "false")
-
-Outputs (pushed to write_repo/write_directory/):
- gha_brief.md — executive action plan
- gha_review.md — combined risk assessment
- gha_publishing.md — workflow classification results
- gha_security.md — all security findings
- gha_publishing_detail.md — enriched: secrets, action SHAs, triggers, run history
- gha_publishing_risks.md — dangerous patterns analysis
- gha_artifact_verification.md — registry cross-reference
- gha_atr_catalog.json — Apache Trusted Releases feed
- gha_channel_pypi.md — per-channel detail (one per active channel)
- gha_channel_npm.md
- gha_channel_maven_central.md
- gha_channel_docker_hub.md
- ...
- gha_json_export.json — machine-readable structured export
-
-Pipeline:
- Phase 1: Prefetch (caches workflow YAML from GitHub API)
- Phase 2: Analysis + Reports
- ├─ Publishing (LLM classification)
- ├─ Security (pattern-matching)
- ├─ Publishing Detail (YAML parsing + GitHub API run history)
- ├─ Artifact Verify (registry API queries)
- ├─ Review (combined risk assessment)
- ├─ Brief (executive summary)
- └─ JSON Export (machine-readable)
+ fresh — "true" to clear all cached data (default "false")
+
+Output structure:
+ {write_directory}/
+ ├── README.md
+ ├── brief.md, review.md, publishing.md, security.md
+ ├── publishing_detail.md, publishing_risks.md, artifact_verification.md
+ ├── json_export.json
+ ├── channels/
+ │ ├── README.md
+ │ └── {channel}.md
+ └── pmcs/
+ ├── README.md
+ └── {pmc}.md
"""
from agent_factory.remote_mcp_client import RemoteMCPClient
@@ -50,7 +36,6 @@ async def run(input_dict, tools):
try:
import json
- # ── Inputs ──
github_owner = input_dict.get("github_owner", "").strip()
read_pat = input_dict.get("read_pat", "").strip()
write_repo = input_dict.get("write_repo", "").strip()
@@ -64,37 +49,9 @@ async def run(input_dict, tools):
if not val:
return {"outputText": f"Error: `{name}` is required."}
- pushed = [] # (filename, status)
+ pushed = []
errors = []
- # ── Fresh: clear all cached data ──
- if fresh:
- NAMESPACES = [
- f"ci-workflows:{github_owner}",
- f"ci-classification:{github_owner}",
- f"ci-report:{github_owner}",
- f"ci-security:{github_owner}",
- f"ci-publishing-detail:{github_owner}",
- f"ci-artifact-verify:{github_owner}",
- f"ci-combined:{github_owner}",
- ]
- print(f"\n{'='*60}", flush=True)
- print(f"Fresh run: clearing all cached data", flush=True)
- print(f"{'='*60}\n", flush=True)
- total_cleared = 0
- for ns_name in NAMESPACES:
- try:
- ns = data_store.use_namespace(ns_name)
- cleared = ns.clear()
- if cleared:
- total_cleared += cleared
- print(f" Cleared {cleared} keys from {ns_name}", flush=True)
- except Exception as e:
- print(f" Could not clear {ns_name}: {e}", flush=True)
- print(f" Total: {total_cleared} keys cleared\n", flush=True)
-
- # ── Helpers ──
-
async def push_file(filename, content, commit_msg):
try:
filepath = f"{write_directory}/{filename}" if write_directory else filename
@@ -136,15 +93,30 @@ async def run(input_dict, tools):
return None
async def push_multi_file_output(output, fallback_name):
- """Parse JSON {files: {name: content}} and push each with gha_ prefix."""
+ """Parse JSON {files: {name: content}} and route to directories."""
if not output:
return
try:
data = json.loads(output)
files = data.get("files", {})
for fname, content in sorted(files.items()):
- gha_name = "gha_" + fname.replace("-", "_")
- await push_file(gha_name, content,
+ # Route channel files to channels/ directory
+ if fname.startswith("channel-"):
+ ch_name = fname.replace("channel-", "").replace(".md", "")
+ target = f"channels/{ch_name}.md"
+ # Route PMC files to pmcs/ directory
+ elif fname.startswith("pmc-"):
+ pmc_name = fname.replace("pmc-", "").replace(".md", "")
+ target = f"pmcs/{pmc_name}.md"
+ # Route index files
+ elif fname == "channels-index.md":
+ target = "channels/README.md"
+ elif fname == "pmcs-index.md":
+ target = "pmcs/README.md"
+ else:
+ # Top-level file, use as-is
+ target = fname
+ await push_file(target, content,
f"CI scan: {fname} ({github_owner})")
return
except (json.JSONDecodeError, AttributeError):
@@ -152,6 +124,96 @@ async def run(input_dict, tools):
await push_file(fallback_name, output,
f"CI scan: {fallback_name} ({github_owner})")
+ def generate_readme(owner):
+ return f"""# GitHub Actions Security Review: {owner}
+
+Automated CI/CD security and publishing analysis. Start with [brief.md](brief.md) for the executive action plan.
+
+## Reports
+
+| Report | Description |
+|--------|-------------|
+| [brief.md](brief.md) | Executive action plan — what to fix first |
+| [review.md](review.md) | Combined risk assessment with per-repo breakdown |
+| [publishing.md](publishing.md) | Workflow classification: release, snapshot, CI, docs |
+| [security.md](security.md) | Org-wide security summary grouped by severity |
+| [publishing_detail.md](publishing_detail.md) | Enriched: secrets, action SHAs, triggers, run history |
+| [publishing_risks.md](publishing_risks.md) | Dangerous publishing patterns |
+| [artifact_verification.md](artifact_verification.md) | Registry cross-reference: verified vs phantom packages |
+
+## Per-Channel Reports
+
+Publishing detail broken down by distribution channel. See [channels/](channels/) for the full list.
+
+| Channel | Report |
+|---------|--------|
+| PyPI | [channels/pypi.md](channels/pypi.md) |
+| npm | [channels/npm.md](channels/npm.md) |
+| Maven Central | [channels/maven_central.md](channels/maven_central.md) |
+| Docker Hub | [channels/docker_hub.md](channels/docker_hub.md) |
+| crates.io | [channels/crates_io.md](channels/crates_io.md) |
+| NuGet | [channels/nuget.md](channels/nuget.md) |
+| GitHub Releases | [channels/github_releases.md](channels/github_releases.md) |
+| Helm | [channels/helm.md](channels/helm.md) |
+| ghcr.io | [channels/ghcr.md](channels/ghcr.md) |
+
+## Per-Project Security Reports
+
+Security findings grouped by ASF project (PMC). See [pmcs/](pmcs/) for all projects.
+
+Each project page tracks finding history across scans — resolved findings are noted
+so progress is visible over time. Projects with no active findings show their
+resolved history.
+
+## Machine-Readable
+
+| File | Description |
+|------|-------------|
+| [json_export.json](json_export.json) | Structured export of all repos, workflows, classifications, findings |
+
+## jq Examples
+
+```bash
+# Top 10 repos by finding count
+jq -r '.repos[] | select(.findings | length > 0) | [.repo, (.findings | length)] | @tsv' json_export.json | sort -t$'\\t' -k2 -rn | head -10
+
+# All CRITICAL findings
+jq -r '.repos[].findings[] | select(.severity == "CRITICAL") | [.check, .file, .detail[:80]] | @tsv' json_export.json
+
+# Repos publishing to PyPI with auth method
+jq -r '.repos[] | select(.ecosystems[] == "pypi") | [.repo, (.workflows[] | select(.publishes and (.ecosystems[] == "pypi")) | .auth_method)] | @tsv' json_export.json
+
+# Trusted publishing migration candidates
+jq -r '.repos[] | select(.trusted_publishing.migration_needed) | [.repo, (.trusted_publishing.eligible_ecosystems | join(","))] | @tsv' json_export.json
+```
+"""
+
+ # ── Fresh: clear all cached data ──
+ if fresh:
+ NAMESPACES = [
+ f"ci-workflows:{github_owner}",
+ f"ci-classification:{github_owner}",
+ f"ci-report:{github_owner}",
+ f"ci-security:{github_owner}",
+ f"ci-publishing-detail:{github_owner}",
+ f"ci-artifact-verify:{github_owner}",
+ f"ci-combined:{github_owner}",
+ ]
+ print(f"\n{'='*60}", flush=True)
+ print(f"Fresh run: clearing all cached data", flush=True)
+ print(f"{'='*60}\n", flush=True)
+ total_cleared = 0
+ for ns_name in NAMESPACES:
+ try:
+ ns = data_store.use_namespace(ns_name)
+ cleared = ns.clear()
+ if cleared:
+ total_cleared += cleared
+ print(f" Cleared {cleared} keys from {ns_name}", flush=True)
+ except Exception as e:
+ print(f" Could not clear {ns_name}: {e}", flush=True)
+ print(f" Total: {total_cleared} keys cleared\n", flush=True)
+
# ══════════════════════════════════════════════════
# Phase 1: Prefetch
# ══════════════════════════════════════════════════
@@ -184,48 +246,52 @@ async def run(input_dict, tools):
publishing_report = await run_agent("gha_publishing",
github_owner=github_owner)
if publishing_report:
- await push_file("gha_publishing.md", publishing_report,
+ await push_file("publishing.md", publishing_report,
f"CI scan: publishing ({github_owner})")
print("▸ Security analysis...", flush=True)
security_report = await run_agent("gha_security",
github_owner=github_owner)
- if security_report:
- await push_file("gha_security.md", security_report,
- f"CI scan: security ({github_owner})")
+ await push_multi_file_output(security_report, "security.md")
print("▸ Publishing detail...", flush=True)
detail_output = await run_agent("gha_publishing_detail",
github_owner=github_owner,
read_pat=read_pat)
- await push_multi_file_output(detail_output, "gha_publishing_detail.md")
+ await push_multi_file_output(detail_output, "publishing_detail.md")
print("▸ Artifact verification...", flush=True)
verify_output = await run_agent("gha_artifact_verify",
github_owner=github_owner)
- await push_multi_file_output(verify_output, "gha_artifact_verification.md")
+ await push_multi_file_output(verify_output, "artifact_verification.md")
print("▸ Combined review...", flush=True)
review_report = await run_agent("gha_review",
github_owner=github_owner)
if review_report:
- await push_file("gha_review.md", review_report,
+ await push_file("review.md", review_report,
f"CI scan: review ({github_owner})")
print("▸ Executive brief...", flush=True)
brief_report = await run_agent("gha_brief",
github_owner=github_owner)
if brief_report:
- await push_file("gha_brief.md", brief_report,
+ await push_file("brief.md", brief_report,
f"CI scan: brief ({github_owner})")
print("▸ JSON export...", flush=True)
json_report = await run_agent("gha_json_export",
github_owner=github_owner)
if json_report:
- await push_file("gha_json_export.json", json_report,
+ await push_file("json_export.json", json_report,
f"CI scan: JSON export ({github_owner})")
+ # ── README ──
+ print("▸ Report index...", flush=True)
+ readme = generate_readme(github_owner)
+ await push_file("README.md", readme,
+ f"CI scan: report index ({github_owner})")
+
# ══════════════════════════════════════════════════
# Summary
# ══════════════════════════════════════════════════
diff --git a/gha-review/agents/gha_publishing_detail.py b/gha-review/agents/gha_publishing_detail.py
index a322bc5..587f58b 100644
--- a/gha-review/agents/gha_publishing_detail.py
+++ b/gha-review/agents/gha_publishing_detail.py
@@ -38,6 +38,8 @@ async def run(input_dict, tools):
['twine upload','python -m twine'], ['pypi.org','upload.pypi.org']),
'test_pypi': ('TestPyPI', [], [], ['test.pypi.org']),
'npm': ('npm', [], ['npm publish'], ['registry.npmjs.org']),
+ 'npm_staging': ('npm (staging)', [], ['npm publish --staging'],
+ ['npm.pkg.github.com', 'verdaccio']),
'maven_central': ('Maven Central', [],
['mvn deploy','mvn -B deploy','./gradlew publish'],
['repo1.maven.org','oss.sonatype.org/service/local/staging',
@@ -89,7 +91,7 @@ async def run(input_dict, tools):
STAGING_CHANNELS = {
'test_pypi','maven_snapshots','apache_dist_dev','nightlies',
- 'ghcr','github_packages','github_pages',
+ 'ghcr','github_packages','github_pages','npm_staging',
}
DOCS_CHANNELS = {'github_pages'}
@@ -315,20 +317,45 @@ async def run(input_dict, tools):
lines.append(f"- **{total_wfs}** workflows analyzed")
lines.append(f"- **{len(all_secrets)}** distinct stored secrets")
lines.append(f"- **{len(unpinned)}** unpinned publishing actions\n")
- lines.append("## Channels\n")
- lines.append("| Channel | Repos | Type |")
- lines.append("|---------|-------|------|")
- for ch in sorted(ch_repos):
- display = CHANNEL_DETECT.get(ch, (ch,))[0]
- ctype = "Production" if ch in PRODUCTION_CHANNELS else "Staging"
- lines.append(f"| {display} | {len(ch_repos[ch])} | {ctype} |")
- lines.append("")
+
+ # Split channels into production and staging
+ prod_chs = {ch: repos for ch, repos in ch_repos.items() if ch in PRODUCTION_CHANNELS}
+ stage_chs = {ch: repos for ch, repos in ch_repos.items() if ch not in PRODUCTION_CHANNELS}
+
+ if prod_chs:
+ lines.append("## Production Channels\n")
+ lines.append("| Channel | Repos | Report |")
+ lines.append("|---------|-------|--------|")
+ for ch in sorted(prod_chs):
+ display = CHANNEL_DETECT.get(ch, (ch,))[0]
+ lines.append(f"| {display} | {len(prod_chs[ch])} | [channels/{ch}.md](channels/{ch}.md) |")
+ lines.append("")
+
+ if stage_chs:
+ lines.append("## Staging Channels\n")
+ lines.append("| Channel | Repos | Report |")
+ lines.append("|---------|-------|--------|")
+ for ch in sorted(stage_chs):
+ display = CHANNEL_DETECT.get(ch, (ch,))[0]
+ lines.append(f"| {display} | {len(stage_chs[ch])} | [channels/{ch}.md](channels/{ch}.md) |")
+ lines.append("")
+
lines.append("## Per-Repository Detail\n")
for repo in sorted(repos, key=lambda r: r['repo']):
lines.append(f"### {repo['repo']}\n")
- lines.append(f"Channels: {', '.join(repo['channels'])}\n")
+ ch_links = [f"[{c}](channels/{c}.md)" for c in repo['channels']]
+ lines.append(f"Channels: {', '.join(ch_links)}\n")
for wf in sorted(repo['workflows'], key=lambda w: w['file']):
- lines.append(f"#### [`{wf['file']}`]({wf['github_url']}) ({wf['category']})\n")
+ # Show meaningful label: category if useful, otherwise primary trigger
+ cat = wf.get('category', '')
+ if cat in ('release_artifact', 'snapshot_artifact'):
+ label = cat.replace('_artifact', '')
+ elif wf.get('triggers'):
+ label = wf['triggers'][0]
+ else:
+ label = ''
+ label_str = f" ({label})" if label else ""
+ lines.append(f"#### [`{wf['file']}`]({wf['github_url']}){label_str}\n")
lines.append(f"- **Triggers:** {', '.join(wf['triggers']) or 'unknown'}")
if wf.get('production_channels'):
lines.append(f"- **Production:** {', '.join(wf['production_channels'])}")
@@ -374,13 +401,17 @@ async def run(input_dict, tools):
lines.append(f"## 1. Pull Request Triggers on Production Publishing ({len(pr_pub)} workflows)\n")
lines.append("**CRITICAL** — fork PRs may access publishing secrets.\n")
for r, f, chs, secs, trigs in sorted(pr_pub):
- lines.append(f"- **{r}** `{f}` → {', '.join(chs)}")
+ wf_url = f"https://github.com/{r}/blob/HEAD/.github/workflows/{f}"
+ lines.append(f"- **[{r}](publishing_detail.md#{r.replace('/', '').lower()})** "
+ f"[`{f}`]({wf_url}) → {', '.join(chs)}")
if secs: lines.append(f" - Secrets: `{'`, `'.join(secs)}`")
lines.append("")
lines.append(f"## 2. Auto-Publish on Push to Default Branch ({len(push_pub)} workflows)\n")
lines.append("**HIGH** — no human gate (tag/release) before production publishing.\n")
for r, f, chs, secs, trigs in sorted(push_pub):
- lines.append(f"- **{r}** `{f}` → {', '.join(chs)}")
+ wf_url = f"https://github.com/{r}/blob/HEAD/.github/workflows/{f}"
+ lines.append(f"- **[{r}](publishing_detail.md#{r.replace('/', '').lower()})** "
+ f"[`{f}`]({wf_url}) → {', '.join(chs)}")
lines.append("")
shared = [(s, rs) for s, rs in sorted(secret_blast.items(), key=lambda x: -len(x[1])) if len(rs) >= 3]
lines.append(f"## 3. Shared Secrets ({len(shared)} names across 3+ repos)\n")
@@ -391,13 +422,17 @@ async def run(input_dict, tools):
lines.append(f"## 4. Cron + Production Secrets ({len(cron_pub)} workflows)\n")
lines.append("**MEDIUM-HIGH** — scheduled runs with publishing credentials.\n")
for r, f, chs, secs, trigs in sorted(cron_pub):
+ wf_url = f"https://github.com/{r}/blob/HEAD/.github/workflows/{f}"
sched = [t for t in trigs if t.startswith('schedule')]
- lines.append(f"- **{r}** `{f}` → {', '.join(chs)} | {', '.join(sched)}")
+ lines.append(f"- **[{r}](publishing_detail.md#{r.replace('/', '').lower()})** "
+ f"[`{f}`]({wf_url}) → {', '.join(chs)} | {', '.join(sched)}")
lines.append("")
lines.append(f"## 5. Semantic-Release ({len(semrel)} workflows)\n")
lines.append("**MEDIUM-HIGH** — auto-version + auto-publish.\n")
for r, f, chs, secs in sorted(semrel):
- lines.append(f"- **{r}** `{f}` → {', '.join(chs)}")
+ wf_url = f"https://github.com/{r}/blob/HEAD/.github/workflows/{f}"
+ lines.append(f"- **[{r}](publishing_detail.md#{r.replace('/', '').lower()})** "
+ f"[`{f}`]({wf_url}) → {', '.join(chs)}")
lines.append("")
return "\n".join(lines)
@@ -463,9 +498,33 @@ async def run(input_dict, tools):
if ch_key in DOCS_CHANNELS:
continue
display = CHANNEL_DETECT.get(ch_key, (ch_key,))[0]
- fname = f"channel-{ch_key.replace('_', '-')}.md"
+ fname = f"channel-{ch_key}.md"
files[fname] = gen_channel_report(ch_key, display, items, enriched_repos, github_owner)
+ # Generate channels index (routed to channels/README.md by orchestrator)
+ ch_index = ["# Publishing Channels\n"]
+ prod_chs = [(k, v) for k, v in ch_wfs.items() if k in PRODUCTION_CHANNELS and k not in DOCS_CHANNELS]
+ stage_chs = [(k, v) for k, v in ch_wfs.items() if k not in PRODUCTION_CHANNELS and k not in DOCS_CHANNELS]
+ if prod_chs:
+ ch_index.append("## Production\n")
+ ch_index.append("| Channel | Repos | Workflows |")
+ ch_index.append("|---------|-------|-----------|")
+ for ch, items in sorted(prod_chs):
+ display = CHANNEL_DETECT.get(ch, (ch,))[0]
+ n_repos = len(set(r for r, _ in items))
+ ch_index.append(f"| [{display}]({ch}.md) | {n_repos} | {len(items)} |")
+ ch_index.append("")
+ if stage_chs:
+ ch_index.append("## Staging\n")
+ ch_index.append("| Channel | Repos | Workflows |")
+ ch_index.append("|---------|-------|-----------|")
+ for ch, items in sorted(stage_chs):
+ display = CHANNEL_DETECT.get(ch, (ch,))[0]
+ n_repos = len(set(r for r, _ in items))
+ ch_index.append(f"| [{display}]({ch}.md) | {n_repos} | {len(items)} |")
+ ch_index.append("")
+ files['channels-index.md'] = "\n".join(ch_index)
+
# Store in CouchDB (one bulk write)
detail_ns = data_store.use_namespace(f"ci-publishing-detail:{github_owner}")
couch_writes = {"latest_data": {
diff --git a/gha-review/agents/gha_security.py b/gha-review/agents/gha_security.py
index 9d96df0..661a230 100644
--- a/gha-review/agents/gha_security.py
+++ b/gha-review/agents/gha_security.py
@@ -895,148 +895,261 @@ async def run(input_dict, tools):
chk = f.get("check", "unknown")
check_counts[chk] = check_counts.get(chk, 0) + 1
- lines = []
- lines.append(f"Analyzed **{repos_scanned}** repositories using cached workflow YAML "
- f"from the Publishing Analyzer.\n")
-
- lines.append("## Executive Summary\n")
- lines.append("| Severity | Count |")
- lines.append("|----------|-------|")
- for sev in ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"]:
- count = severity_counts.get(sev, 0)
- if count > 0:
- lines.append(f"| **{sev}** | **{count}** |")
- lines.append("")
-
- check_descriptions = {
- "prt_checkout": "pull_request_target + checkout (CRITICAL only when ref: points to PR head)",
- "self_hosted_runner": "Self-hosted runners exposed to PR triggers",
- "broad_permissions": "Overly broad GITHUB_TOKEN permissions",
- "cache_poisoning": "Potential cache poisoning via PR-controlled keys",
- "run_block_injection": "Direct ${{ }} interpolation in workflow run blocks",
- "unpinned_actions": "Third-party actions not SHA-pinned (ASF policy violation)",
- "third_party_actions": "Actions from outside actions/*/github/*/apache/* namespaces",
- "missing_codeowners": "No CODEOWNERS file",
- "missing_dependency_updates": "No dependabot/renovate configuration (ASF policy violation)",
- "composite_action_injection": "Injection in composite action run block",
- "composite_action_unpinned": "Third-party unpinned action ref inside composite action",
- "composite_action_input_injection": "Latent injection surface — composite action interpolates inputs.* in run block",
- }
+ # ── Report generation ──
+ import hashlib
+
+ EXCLUDE_FROM_REPORT = {'missing_codeowners'}
+ SEV_ORDER_MAP = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4}
+
+ def extract_pmc(repo_name):
+ name = repo_name
+ if name.startswith('incubator-'):
+ name = name[len('incubator-'):]
+ return name.split('-')[0]
+
+ def wf_link(repo, f):
+ """GitHub link to workflow file, with line number if available."""
+ file = f.get("file", "")
+ base = f"https://github.com/{github_owner}/{repo}/blob/HEAD/.github/workflows/{file}"
+ if f.get("line"):
+ return f"[{file}#L{f['line']}]({base}#L{f['line']})"
+ return f"[{file}]({base})"
+
+ def finding_hash(f):
+ """Stable identity for a finding across runs."""
+ key = f"{f.get('check','')}:{f.get('file','')}:{f.get('detail','')[:60]}"
+ return hashlib.md5(key.encode()).hexdigest()[:12]
+
+ # Filter out excluded checks from display
+ display_findings = {}
+ for repo, findings in all_findings.items():
+ filtered = [f for f in findings if f.get("check") not in EXCLUDE_FROM_REPORT]
+ if filtered:
+ display_findings[repo] = filtered
- lines.append("## Findings by Check Type\n")
- lines.append("| Check | Count | Description |")
- lines.append("|-------|-------|-------------|")
- for chk, count in sorted(check_counts.items(), key=lambda x: -x[1]):
- desc = check_descriptions.get(chk, chk)
- lines.append(f"| {chk} | {count} | {desc} |")
- lines.append("")
+ # Severity counts (display only)
+ severity_counts = {}
+ check_counts = {}
+ for repo, findings in display_findings.items():
+ for f in findings:
+ sev = f.get("severity", "INFO")
+ severity_counts[sev] = severity_counts.get(sev, 0) + 1
+ chk = f.get("check", "unknown")
+ check_counts[chk] = check_counts.get(chk, 0) + 1
+ total_findings = sum(severity_counts.values())
+ # Group by severity
by_severity = {"CRITICAL": [], "HIGH": [], "MEDIUM": [], "LOW": [], "INFO": []}
- for repo, findings in all_findings.items():
+ for repo, findings in display_findings.items():
for f in findings:
sev = f.get("severity", "INFO")
if sev in by_severity:
by_severity[sev].append((repo, f))
- def file_ref(f):
- """Format file reference with line number(s) if available."""
- name = f.get("file", "")
- if f.get("lines"):
- return f"{name}:{','.join(str(l) for l in f['lines'])}"
- elif f.get("line"):
- return f"{name}:{f['line']}"
- return name
-
- if by_severity["CRITICAL"]:
- lines.append("## CRITICAL Findings\n")
- lines.append("Untrusted external input directly interpolated in shell execution contexts.\n")
- for repo, f in sorted(by_severity["CRITICAL"], key=lambda x: (x[0], x[1].get("file", ""))):
- lines.append(f"- **{github_owner}/{repo}** (`{file_ref(f)}`): [{f['check']}] {f['detail']}")
- lines.append("")
-
- if by_severity["HIGH"]:
- lines.append("## HIGH Findings\n")
- for repo, f in sorted(by_severity["HIGH"], key=lambda x: (x[0], x[1].get("file", ""))):
- lines.append(f"- **{github_owner}/{repo}** (`{file_ref(f)}`): [{f['check']}] {f['detail']}")
- lines.append("")
-
- if by_severity["MEDIUM"]:
- lines.append("## MEDIUM Findings\n")
- lines.append(f"<details>\n<summary>Show {len(by_severity['MEDIUM'])} medium findings</summary>\n")
- for repo, f in sorted(by_severity["MEDIUM"], key=lambda x: (x[0], x[1].get("file", ""))):
- lines.append(f"- **{github_owner}/{repo}** (`{file_ref(f)}`): [{f['check']}] {f['detail']}")
- lines.append(f"\n</details>\n")
-
- if by_severity["LOW"]:
- lines.append("## LOW Findings\n")
- lines.append(f"<details>\n<summary>Show {len(by_severity['LOW'])} low findings</summary>\n")
- for repo, f in sorted(by_severity["LOW"], key=lambda x: (x[0], x[1].get("file", ""))):
- lines.append(f"- **{github_owner}/{repo}** (`{file_ref(f)}`): [{f['check']}] {f['detail']}")
- lines.append(f"\n</details>\n")
-
- if by_severity["INFO"]:
- lines.append("## INFO Findings\n")
- lines.append(f"<details>\n<summary>Show {len(by_severity['INFO'])} info findings</summary>\n")
- for repo, f in sorted(by_severity["INFO"], key=lambda x: (x[0], x[1].get("file", ""))):
- lines.append(f"- **{github_owner}/{repo}** (`{file_ref(f)}`): [{f['check']}] {f['detail']}")
- lines.append(f"\n</details>\n")
-
- lines.append("## Detailed Results by Repository\n")
- for repo in sorted(all_findings.keys()):
- findings = all_findings[repo]
- if not findings:
- continue
- sev_summary = {}
- for f in findings:
- sev_summary[f["severity"]] = sev_summary.get(f["severity"], 0) + 1
- sev_str = ", ".join(f"{s}: {c}" for s, c in sorted(sev_summary.items()))
-
- lines.append(f"### {github_owner}/{repo}\n")
- lines.append(f"**{len(findings)}** findings | {sev_str}\n")
-
- sev_order = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4}
- for f in sorted(findings, key=lambda x: sev_order.get(x["severity"], 99)):
- lines.append(f"- **[{f['severity']}]** `{file_ref(f)}` — [{f['check']}] {f['detail']}")
- lines.append("")
-
- lines.append("---\n")
- lines.append(f"*Findings cached in `ci-security:{github_owner}`.*")
+ # Group by PMC
+ pmc_findings = {} # pmc -> {repo -> [findings]}
+ for repo, findings in display_findings.items():
+ pmc = extract_pmc(repo)
+ pmc_findings.setdefault(pmc, {}).setdefault(repo, []).extend(findings)
+
+ # ── Progress tracking ──
+ history_ns = data_store.use_namespace(f"ci-security-history:{github_owner}")
+ previous_raw = None
+ try:
+ previous_raw = history_ns.get("previous_findings")
+ except Exception:
+ pass
+ previous = {}
+ if previous_raw and isinstance(previous_raw, dict):
+ previous = previous_raw # {repo: [hash1, hash2, ...]}
+
+ # Compute current hashes
+ current_hashes = {} # {repo: [hash1, ...]}
+ for repo, findings in display_findings.items():
+ current_hashes[repo] = [finding_hash(f) for f in findings]
+
+ # Find resolved findings
+ resolved_raw = None
+ try:
+ resolved_raw = history_ns.get("resolved_findings")
+ except Exception:
+ pass
+ resolved_all = resolved_raw if isinstance(resolved_raw, list) else []
+
+ # New resolutions this run
+ from datetime import datetime, timezone as tz_mod
+ run_date = datetime.now(tz_mod.utc).strftime("%Y-%m-%d")
+ for repo, prev_hashes in previous.items():
+ curr = set(current_hashes.get(repo, []))
+ for h in prev_hashes:
+ if h not in curr:
+ resolved_all.append({"hash": h, "repo": repo, "resolved_date": run_date})
+
+ # Save current state for next run
+ if not redacted_severity:
+ try:
+ history_ns.set_many({
+ "previous_findings": current_hashes,
+ "resolved_findings": resolved_all,
+ })
+ except Exception as e:
+ print(f" Warning: could not save history: {e}", flush=True)
- report_body = "\n".join(lines)
+ # Resolved findings by PMC
+ resolved_by_pmc = {}
+ for r in resolved_all:
+ pmc = extract_pmc(r.get("repo", ""))
+ resolved_by_pmc.setdefault(pmc, []).append(r)
- def to_anchor(text):
- anchor = text.lower().strip()
- anchor = re.sub(r'[^\w\s-]', '', anchor)
- anchor = re.sub(r'\s+', '-', anchor)
- anchor = re.sub(r'-+', '-', anchor)
- return anchor.strip('-')
+ # ── Generate main security.md (summary linking to PMC pages) ──
+ lines = []
+ lines.append(f"# Security Analysis: {github_owner}\n")
+ lines.append(f"**{len(display_findings)}** repos with findings, "
+ f"**{total_findings}** total findings across "
+ f"**{len(pmc_findings)}** projects.\n")
- toc_lines = [f"# {report_title}\n", "## Contents\n"]
- toc_lines.append(f"- [Executive Summary](#{to_anchor('Executive Summary')})")
- toc_lines.append(f"- [Findings by Check Type](#{to_anchor('Findings by Check Type')})")
+ lines.append("## Summary by Severity\n")
for sev in ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"]:
- if by_severity.get(sev):
- toc_lines.append(f"- [{sev} Findings](#{to_anchor(f'{sev} Findings')}) ({len(by_severity[sev])})")
- toc_lines.append(f"- [Detailed Results](#{to_anchor('Detailed Results by Repository')})")
- for repo in sorted(all_findings.keys()):
- toc_lines.append(f" - [{github_owner}/{repo}](#{to_anchor(f'{github_owner}/{repo}')})")
+ items = by_severity.get(sev, [])
+ if not items:
+ continue
+ lines.append(f"### {sev} ({len(items)} findings)\n")
+ # Group by check type within severity
+ by_check = {}
+ for repo, f in items:
+ chk = f.get("check", "unknown")
+ by_check.setdefault(chk, []).append((repo, f))
+ for chk, chk_items in sorted(by_check.items(), key=lambda x: -len(x[1])):
+ # Dedupe to PMC level
+ affected_pmcs = sorted(set(extract_pmc(r) for r, _ in chk_items))
+ pmc_links = ', '.join(f"[{p}](pmcs/{p}.md)" for p in affected_pmcs)
+ lines.append(f"**{chk}** ({len(chk_items)} findings) — {pmc_links}\n")
+ lines.append("")
- toc = "\n".join(toc_lines)
- full_report = toc + "\n\n---\n\n" + report_body
+ lines.append("## Projects\n")
+ lines.append("| Project | Findings | Worst Severity | Report |")
+ lines.append("|---------|----------|---------------|--------|")
+ for pmc in sorted(pmc_findings.keys()):
+ repos = pmc_findings[pmc]
+ n = sum(len(fs) for fs in repos.values())
+ worst = "INFO"
+ for fs in repos.values():
+ for f in fs:
+ if SEV_ORDER_MAP.get(f.get("severity","INFO"), 4) < SEV_ORDER_MAP.get(worst, 4):
+ worst = f["severity"]
+ lines.append(f"| {pmc} | {n} | {worst} | [pmcs/{pmc}.md](pmcs/{pmc}.md) |")
+ lines.append("")
+ lines.append("---\n")
+ lines.append(f"*{repos_scanned} repos scanned, {total_findings} findings.*")
+
+ report_title = f"Security Analysis: {github_owner}"
+
+ # ── Build multi-file output ──
+ files = {}
+ files['security.md'] = "\n".join(lines)
+
+ # ── Per-PMC pages ──
+ for pmc, repos in sorted(pmc_findings.items()):
+ pmc_lines = []
+ n_findings = sum(len(fs) for fs in repos.values())
+ pmc_lines.append(f"# {pmc}\n")
+ pmc_lines.append(f"**{len(repos)}** repos, **{n_findings}** active findings.\n")
+
+ # Group PMC findings by severity
+ pmc_by_sev = {"CRITICAL": [], "HIGH": [], "MEDIUM": [], "LOW": [], "INFO": []}
+ for repo, findings in repos.items():
+ for f in findings:
+ sev = f.get("severity", "INFO")
+ if sev in pmc_by_sev:
+ pmc_by_sev[sev].append((repo, f))
+
+ for sev in ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"]:
+ items = pmc_by_sev[sev]
+ if not items:
+ continue
+ pmc_lines.append(f"## {sev}\n")
+ for repo, f in sorted(items, key=lambda x: (x[0], x[1].get("file",""))):
+ pmc_lines.append(f"- **{github_owner}/{repo}** {wf_link(repo, f)} — "
+ f"[{f['check']}] {f['detail']}")
+ pmc_lines.append("")
+
+ # Resolved findings for this PMC
+ pmc_resolved = resolved_by_pmc.get(pmc, [])
+ if pmc_resolved:
+ pmc_lines.append("## Resolved\n")
+ for r in sorted(pmc_resolved, key=lambda x: x.get("resolved_date", "")):
+ pmc_lines.append(f"- ~~{r.get('repo', '?')}~~ resolved {r.get('resolved_date', '?')}")
+ pmc_lines.append("")
+
+ files[f"pmc-{pmc}.md"] = "\n".join(pmc_lines)
+
+ # PMC pages for projects with ONLY resolved findings (all clear)
+ for pmc, resolved_list in resolved_by_pmc.items():
+ if pmc not in pmc_findings:
+ clear_lines = [f"# {pmc}\n",
+ "All clear. No active security findings.\n",
+ "## Resolved\n"]
+ for r in sorted(resolved_list, key=lambda x: x.get("resolved_date", "")):
+ clear_lines.append(f"- ~~{r.get('repo', '?')}~~ resolved {r.get('resolved_date', '?')}")
+ clear_lines.append("")
+ files[f"pmc-{pmc}.md"] = "\n".join(clear_lines)
+
+ # ── PMC index ──
+ pmcs_index = ["# Project Security Reports\n",
+ f"**{len(pmc_findings)}** projects with active findings.\n"]
+ pmcs_index.append("| Project | Repos | Findings | Worst | Report |")
+ pmcs_index.append("|---------|-------|----------|-------|--------|")
+ # Sort by worst severity, then finding count
+ def pmc_sort_key(pmc):
+ repos = pmc_findings.get(pmc, {})
+ worst = 4
+ n = 0
+ for fs in repos.values():
+ n += len(fs)
+ for f in fs:
+ s = SEV_ORDER_MAP.get(f.get("severity","INFO"), 4)
+ if s < worst:
+ worst = s
+ return (worst, -n)
+
+ for pmc in sorted(pmc_findings.keys(), key=pmc_sort_key):
+ repos = pmc_findings[pmc]
+ n = sum(len(fs) for fs in repos.values())
+ worst = "INFO"
+ for fs in repos.values():
+ for f in fs:
+ if SEV_ORDER_MAP.get(f.get("severity","INFO"), 4) < SEV_ORDER_MAP.get(worst, 4):
+ worst = f["severity"]
+ pmcs_index.append(f"| {pmc} | {len(repos)} | {n} | {worst} | [{pmc}.md]({pmc}.md) |")
+ pmcs_index.append("")
+ files["pmcs-index.md"] = "\n".join(pmcs_index)
+
+ print(f"Generated {len(files)} security report files ({len(pmc_findings)} PMC pages)", flush=True)
+
+ # Store stats (original findings including codeowners, for json_export compatibility)
if not redacted_severity:
+ all_severity_counts = {}
+ all_check_counts = {}
+ for repo, findings in all_findings.items():
+ for f in findings:
+ sev = f.get("severity", "INFO")
+ all_severity_counts[sev] = all_severity_counts.get(sev, 0) + 1
+ chk = f.get("check", "unknown")
+ all_check_counts[chk] = all_check_counts.get(chk, 0) + 1
+
security_ns.set_many({
- "latest_report": full_report,
+ "latest_report": files['security.md'],
"latest_stats": {
"repos_scanned": repos_scanned,
"repos_with_findings": len(all_findings),
- "total_findings": total_findings,
- "severity_counts": severity_counts,
- "check_counts": check_counts,
+ "total_findings": sum(all_severity_counts.values()),
+ "severity_counts": all_severity_counts,
+ "check_counts": all_check_counts,
},
})
- return {"outputText": full_report}
+ return {"outputText": json.dumps({"files": files})}
finally:
await http_client.aclose()
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org