You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@incubator.apache.org by jm...@apache.org on 2025/10/16 01:28:30 UTC
(incubator) 01/02: improved script
This is an automated email from the ASF dual-hosted git repository.
jmclean pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator.git
commit 16fefca41581375923895befebd3c26480cdc080
Author: Justin Mclean <jm...@apache.org>
AuthorDate: Thu Oct 16 12:25:00 2025 +1100
improved script
---
tools/6MonthReview/6month_report.py | 536 ++++++++++++++++++------------------
1 file changed, 262 insertions(+), 274 deletions(-)
diff --git a/tools/6MonthReview/6month_report.py b/tools/6MonthReview/6month_report.py
index 181b5ffff..4f274bbed 100644
--- a/tools/6MonthReview/6month_report.py
+++ b/tools/6MonthReview/6month_report.py
@@ -1,58 +1,41 @@
#!/usr/bin/env python3
# podling_6mo_values.py
# Generate <=6mo podling metrics (CSV/MD).
-#
-# Note that this code is mostly AI generated.
-#
-
-import argparse
-import csv
-import os
-import sys
-import time
-import re
-import glob
-import email
+
+import argparse, csv, os, sys, re, glob, email, json
from email.parser import BytesParser
from email.policy import default as EMAIL_DEFAULT
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timezone, timedelta, date
from dateutil import parser as dtp
from typing import Dict, Any, Optional, Tuple, List, Set
from urllib.parse import urlparse, parse_qs
+import xml.etree.ElementTree as ET
import requests
from tabulate import tabulate
# ---------------- Constants ----------------
+INCUBATOR_ROOT = "https://incubator.apache.org"
+URL_PODLINGS_XML = f"{INCUBATOR_ROOT}/podlings.xml"
+
W_PUBLIC = "https://whimsy.apache.org/public"
-URL_PODLINGS = f"{W_PUBLIC}/public_podlings.json"
-URL_PODLING_STATUS = f"{W_PUBLIC}/public_podling_status.json"
URL_LDAP_PROJECTS = f"{W_PUBLIC}/public_ldap_projects.json"
TODAY = datetime.now(timezone.utc).date()
-# Stable mailing list API (mbox)
LAA_MBOX = "https://lists.apache.org/api/mbox.lua"
-# Subjects
+GITHUB_API = "https://api.github.com"
+GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
+EXCLUDE_SUFFIXES = ("-site","-website","-site-pub","-asf-site","-asf-site-src",
+ "-images","-logos","-branding","-docs","-documentation","-examples")
+
RELEASE_VOTE_SUBJECT = re.compile(r"\[(?:VOTE)\].*\b(RELEASE|INCUBATING)\b", re.I)
SUBJ_PREFIX_RE = re.compile(r'^\s*(re|fwd|fw)\s*:\s*', re.I)
-
-# Growth heuristics on general@incubator
PPMC_VOTE_SUBJECT = re.compile(r"\[(?:VOTE|RESULT)\].*(PPMC|PMC).*(member|add|appoint)", re.I)
COMMITTER_VOTE_SUBJECT = re.compile(r"\[(?:VOTE|RESULT)\].*committer", re.I)
-# GitHub
-GITHUB_API = "https://api.github.com"
-GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
-
-# Repo discovery filters (avoid websites/docs-only)
-EXCLUDE_SUFFIXES = (
- "-site", "-website", "-site-pub", "-asf-site", "-asf-site-src",
- "-images", "-logos", "-branding", "-docs", "-documentation", "-examples"
-)
-
# ---------------- HTTP helpers ----------------
def http_json(url: str, params: Optional[Dict[str, Any]] = None,
@@ -160,7 +143,56 @@ def to_int(x, default=0):
except Exception:
return default
-# ---------------- Mailing lists (mbox) ----------------
+def parse_iso_dt(s: Optional[str]) -> Optional[datetime]:
+ if not s:
+ return None
+ try:
+ return dtp.parse(s)
+ except Exception:
+ return None
+
+# ---------------- podlings.xml parsing ----------------
+
+def load_podlings_xml(url: str = URL_PODLINGS_XML) -> Dict[str, Dict[str, Any]]:
+ xml_text = http_text(url)
+ root = ET.fromstring(xml_text)
+ podlings: Dict[str, Dict[str, Any]] = {}
+ for p in root.findall("podling"):
+ resource = (p.get("resource") or "").strip()
+ name = (p.get("name") or resource or "").strip()
+ status = (p.get("status") or "").strip()
+ startdate = (p.get("startdate") or p.get("start_date") or "").strip()
+ enddate = (p.get("enddate") or "").strip()
+
+ mentors: List[Dict[str, str]] = []
+ for m in p.findall("./mentors/mentor"):
+ mentors.append({
+ "name": (m.text or "").strip() or (m.get("name") or "").strip(),
+ "username": (m.get("username") or m.get("availid") or "").strip(),
+ "role": (m.get("role") or "").strip(),
+ })
+
+ # aliases
+ aliases: List[str] = []
+ for ra in p.findall("resourceAlias"):
+ val = (ra.get("name") or (ra.text or "")).strip()
+ if val:
+ aliases.append(val)
+
+ if not resource:
+ continue
+ podlings[resource] = {
+ "id": resource,
+ "name": name,
+ "status": status,
+ "startdate": startdate,
+ "enddate": enddate,
+ "mentors": mentors,
+ "aliases": aliases,
+ }
+ return podlings
+
+# ---------------- Public lists (ancillary signals) ----------------
def fetch_month_mbox(domain: str, listname: str, year: int, month: int, debug: bool=False) -> str:
dval = f"{year:04d}-{month:02d}"
@@ -189,18 +221,14 @@ def iter_mbox_messages(mbox_text: str):
msg[k.strip()] = v.strip()
yield msg
-def infer_dev_list_domain(pod_id: str, meta: Dict[str, Any]) -> str:
+def infer_dev_list_domain(pod_id: str) -> str:
return f"{pod_id}.apache.org"
def lists_activity_dev(dev_domain: str, days: int, debug: bool=False) -> Tuple[int, int, int, List[str]]:
- """
- Returns: (unique_posters, release_vote_threads, total_emails, sample_senders)
- """
unique_senders: Set[str] = set()
vote_threads: Set[str] = set()
sample_senders: List[str] = []
email_count = 0
-
for (y, m) in months_covering_last_days(days):
mbox = fetch_month_mbox(dev_domain, "dev", y, m, debug)
if not mbox:
@@ -216,14 +244,12 @@ def lists_activity_dev(dev_domain: str, days: int, debug: bool=False) -> Tuple[i
unique_senders.add(addr)
if RELEASE_VOTE_SUBJECT.search(subj or ""):
vote_threads.add(canon_subject(subj))
-
return (len(unique_senders), len(vote_threads), email_count, sample_senders)
def lists_release_votes_general(pod_id: str, pod_name: str, days: int, debug: bool=False) -> int:
name_pat = re.compile(re.escape(pod_name), re.I)
id_pat = re.compile(rf"\b{re.escape(pod_id)}\b", re.I)
seen_threads: Set[str] = set()
-
for (y, m) in months_covering_last_days(days):
mbox = fetch_month_mbox("incubator.apache.org", "general", y, m, debug)
if not mbox:
@@ -232,7 +258,6 @@ def lists_release_votes_general(pod_id: str, pod_name: str, days: int, debug: bo
subj = str(msg.get("Subject", "") or "")
if subj and RELEASE_VOTE_SUBJECT.search(subj) and (name_pat.search(subj) or id_pat.search(subj)):
seen_threads.add(canon_subject(subj))
-
return len(seen_threads)
def lists_growth_general(pod_id: str, pod_name: str, days: int, debug: bool=False) -> Tuple[int, int]:
@@ -240,7 +265,6 @@ def lists_growth_general(pod_id: str, pod_name: str, days: int, debug: bool=Fals
id_pat = re.compile(rf"\b{re.escape(pod_id)}\b", re.I)
ppmc_threads: Set[str] = set()
comm_threads: Set[str] = set()
-
for (y, m) in months_covering_last_days(days):
mbox = fetch_month_mbox("incubator.apache.org", "general", y, m, debug)
if not mbox:
@@ -255,30 +279,52 @@ def lists_growth_general(pod_id: str, pod_name: str, days: int, debug: bool=Fals
ppmc_threads.add(norm_s)
if COMMITTER_VOTE_SUBJECT.search(subj):
comm_threads.add(norm_s)
-
return len(ppmc_threads), len(comm_threads)
# ---------------- Mentor reports/sign-offs (board agendas) ----------------
-H2_THIS_POD_RE = lambda pod_id, pod_name: re.compile(
- rf'(?mi)^\s*##\s+(?:{re.escape(pod_id)}|{re.escape(pod_name)})\s*$'
-)
-SIGNED_OFF_RE = re.compile(r'(?mi)^\s*#+\s*Signed[- ]off[- ]by\s*:\s*$')
-BLOCK_END_RE = re.compile(r'(?m)^(?:##\s+|###\s+|-{5,}\s*$)') # next H2/H3 or dashed rule
-CHECKBOX_RE = re.compile(r'\[(?:x|X)\]')
-
-def count_reports_and_signoffs(pod_id: str, pod_name: str, minutes_dir: str, debug: bool=False) -> Tuple[int, int]:
- """
- reports_count: +1 per agenda file that contains an H2 header '## <pod_id|pod_name>'
- mentor_signoffs: number of [X]/[x] checkboxes inside the pod's section, within any
- 'Signed-off-by:' block(s), up to the next '##'/'###' or dashed rule.
- """
+HDR_RE = re.compile(r'(?m)^(?P<hashes>#{1,6})\s+(?P<title>.+?)\s*$')
+SIGNED_OFF_RE = re.compile(r'(?mi)^\s*###\s+Signed-off-by:\s*$') # exact label; case-insensitive
+NEXT_H3_RE = re.compile(r'(?m)^\s*###\s+')
+CHECKED_BULLET_RE = re.compile(r'(?m)^\s*-\s*\[\s*[xX]\s*\]') # "- [X]" lines only
+
+AGENDA_DATE_RE = re.compile(r'^board_agenda_(\d{4})_(\d{2})_(\d{2})\.txt$')
+
+def _agenda_file_date(path: str) -> Optional[date]:
+ m = AGENDA_DATE_RE.match(os.path.basename(path))
+ if not m:
+ return None
+ try:
+ y, mth, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
+ return date(y, mth, d)
+ except Exception:
+ return None
+
+def _section_end_for_level(text: str, start: int, level: int) -> int:
+ for m in HDR_RE.finditer(text, start):
+ next_level = len(m.group('hashes'))
+ if next_level <= level:
+ return m.start()
+ return len(text)
+
+def _match_pod_title(title: str, pod_id: str, pod_name: str) -> bool:
+ t = title.strip().lower()
+ pid = re.escape(pod_id.strip().lower())
+ pname = re.escape(pod_name.strip().lower())
+ return bool(re.search(rf'\b({pid}|{pname})\b', t))
+
+def count_reports_and_signoffs(pod_id: str, pod_name: str, minutes_dir: str,
+ window_days: int = 180, debug: bool=False) -> Tuple[int, int]:
if not minutes_dir:
return 0, 0
-
reports = 0
total_signoffs = 0
+
for path in sorted(glob.glob(os.path.join(minutes_dir, "board_agenda_*.txt"))):
+ fdt = _agenda_file_date(path)
+ if fdt and (TODAY - fdt) > timedelta(days=window_days):
+ continue
+
try:
with open(path, encoding="utf-8", errors="ignore") as f:
txt = f.read()
@@ -287,49 +333,59 @@ def count_reports_and_signoffs(pod_id: str, pod_name: str, minutes_dir: str, deb
print(f"! error reading {path}: {e}", file=sys.stderr)
continue
- section_re = H2_THIS_POD_RE(pod_id, pod_name)
- sec_match = section_re.search(txt)
- if not sec_match:
+ # Find the podling header (any level), then slice its section.
+ pod_hdr = None
+ for m in HDR_RE.finditer(txt):
+ if _match_pod_title(m.group('title'), pod_id, pod_name):
+ pod_hdr = m
+ break
+
+ if not pod_hdr:
if debug:
- print(f"{os.path.basename(path)}: no section for {pod_id}")
+ print(f"{os.path.basename(path)}: podling header: NO")
continue
reports += 1
- sec_start = sec_match.end()
- next_h2 = re.search(r'(?m)^\s*##\s+', txt[sec_start:])
- sec_end = sec_start + (next_h2.start() if next_h2 else 0) if next_h2 else len(txt)
- section = txt[sec_start:sec_end]
+
+ level = len(pod_hdr.group('hashes'))
+ sec_start = pod_hdr.end()
+ sec_end = _section_end_for_level(txt, sec_start, level)
+ pod_section = txt[sec_start:sec_end]
local_signoffs = 0
- for sm in SIGNED_OFF_RE.finditer(section):
- bstart = sm.end()
- bend_m = BLOCK_END_RE.search(section, bstart)
- bend = bend_m.start() if bend_m else len(section)
- block = section[bstart:bend]
- local_signoffs += len(CHECKBOX_RE.findall(block))
+ for sm in SIGNED_OFF_RE.finditer(pod_section):
+ block = pod_section[sm.end():]
+ m_next = NEXT_H3_RE.search(block)
+ if m_next:
+ block = block[:m_next.start()]
+ local_signoffs += len(CHECKED_BULLET_RE.findall(block))
+
+ if debug:
+ hdr_end = pod_section.find("\n", sm.start())
+ hdr_line = pod_section[sm.start(): hdr_end if hdr_end != -1 else len(pod_section)]
+ print(f" Found header: {hdr_line.rstrip()}")
+ for ln in block.splitlines():
+ if CHECKED_BULLET_RE.search(ln):
+ print(f" COUNTED: {ln.rstrip()}")
- total_signoffs += local_signoffs
if debug:
print(f"{os.path.basename(path)}: report=1, signoffs+={local_signoffs}")
- return reports, total_signoffs
+ total_signoffs += local_signoffs
-# --------- Audit mode (show exactly what’s counted) ---------
+ return reports, total_signoffs
-def audit_minutes_for_pod(pod_id: str, pod_name: str, minutes_dir: str) -> None:
- """
- Prints an audit trail: for each board_agenda_*.txt
- - whether ## <pod> section found
- - each Signed-off-by block within that section
- - each [X] line counted
- """
+def audit_minutes_for_pod(pod_id: str, pod_name: str, minutes_dir: str,
+ window_days: int = 180) -> None:
if not minutes_dir:
print("No minutes_dir provided.")
return
- H2_RE = H2_THIS_POD_RE(pod_id, pod_name)
-
for path in sorted(glob.glob(os.path.join(minutes_dir, "board_agenda_*.txt"))):
+ fdt = _agenda_file_date(path)
+ if fdt and (TODAY - fdt) > timedelta(days=window_days):
+ continue
+
try:
with open(path, encoding="utf-8", errors="ignore") as f:
txt = f.read()
@@ -337,34 +393,36 @@ def audit_minutes_for_pod(pod_id: str, pod_name: str, minutes_dir: str) -> None:
print(f"{os.path.basename(path)}: ! read error: {e}")
continue
- sec = H2_RE.search(txt)
- if not sec:
- print(f"{os.path.basename(path)}: section: NO")
+ pod_hdr = None
+ for m in HDR_RE.finditer(txt):
+ if _match_pod_title(m.group('title'), pod_id, pod_name):
+ pod_hdr = m
+ break
+
+ if not pod_hdr:
+ print(f"{os.path.basename(path)}: podling header: NO")
continue
- sec_start = sec.end()
- next_h2 = re.search(r'(?m)^\s*##\s+', txt[sec_start:])
- sec_end = sec_start + (next_h2.start() if next_h2 else 0) if next_h2 else len(txt)
- section = txt[sec_start:sec_end]
- print(f"{os.path.basename(path)}: section: YES (bytes {sec_start}-{sec_end})")
+ level = len(pod_hdr.group('hashes'))
+ sec_start = pod_hdr.end()
+ sec_end = _section_end_for_level(txt, sec_start, level)
+ pod_section = txt[sec_start:sec_end]
+ print(f"{os.path.basename(path)}: podling subsection: YES (bytes {sec_start}-{sec_end})")
total_block = 0
- for sm in SIGNED_OFF_RE.finditer(section):
- bstart = sm.end()
- bend_m = BLOCK_END_RE.search(section, bstart)
- bend = bend_m.start() if bend_m else len(section)
- block = section[bstart:bend]
- lines = [ln.rstrip() for ln in block.splitlines() if CHECKBOX_RE.search(ln)]
+ for sm in SIGNED_OFF_RE.finditer(pod_section):
+ block = pod_section[sm.end():]
+ m_next = NEXT_H3_RE.search(block)
+ if m_next:
+ block = block[:m_next.start()]
+ lines = [ln.rstrip() for ln in block.splitlines() if CHECKED_BULLET_RE.search(ln)]
total_block += len(lines)
print(" Signed-off-by block:")
for ln in lines:
print(f" COUNTED: {ln}")
print(f" => mentor_signoffs in file: {total_block}")
-# ---------------- GitHub helpers ----------------
-
-class Github422(Exception):
- pass
+# ---------------- GitHub helpers (unchanged; optional) ----------------
def gh_repo_exists_full(full: str) -> bool:
try:
@@ -374,14 +432,9 @@ def gh_repo_exists_full(full: str) -> bool:
return False
def gh_repo_info(owner_repo: str) -> Dict[str, Any]:
- """Fetch repository metadata (used to respect has_issues)."""
return http_json(f"{GITHUB_API}/repos/{owner_repo}")
def gh_count_via_link_header(url: str, params: Dict[str, Any]) -> int:
- """
- Count items using per_page=1 + Link rel=last heuristic (works for /pulls).
- Returns 0 on 422 or if the endpoint doesn’t support paging.
- """
p = dict(params or {})
p.setdefault("per_page", 1)
headers = {"Accept": "application/vnd.github+json"}
@@ -391,7 +444,6 @@ def gh_count_via_link_header(url: str, params: Dict[str, Any]) -> int:
if r.status_code == 422:
return 0
r.raise_for_status()
-
link = r.headers.get("Link", "")
if not link:
try:
@@ -399,7 +451,6 @@ def gh_count_via_link_header(url: str, params: Dict[str, Any]) -> int:
return len(data) if isinstance(data, list) else 0
except Exception:
return 0
-
last_page = 1
for part in link.split(","):
if 'rel="last"' in part:
@@ -416,9 +467,6 @@ def gh_count_via_link_header(url: str, params: Dict[str, Any]) -> int:
return last_page
def gh_search_count_safe(query: str) -> int:
- """
- Use GitHub search API for counts; on errors return 0 instead of raising.
- """
try:
url = f"{GITHUB_API}/search/issues"
data = http_json(url, params={"q": query})
@@ -444,41 +492,22 @@ def gh_repo_contributors(owner_repo: str) -> Set[str]:
return ids
def gh_repo_issue_pr_breakdown(owner_repo: str, debug: bool=False) -> Tuple[int, int, int, int]:
- """
- Returns (open_issues, closed_issues, open_prs, closed_prs).
- - PRs counted via /pulls and Link rel=last.
- - Issues counted via search/issues; if search fails or has_issues=False → 0/0.
- """
- # PR counts
try:
open_prs = gh_count_via_link_header(f"{GITHUB_API}/repos/{owner_repo}/pulls", {"state": "open"})
- except Exception as e:
- if debug:
- print(f" · PR open count fallback error for {owner_repo}: {e}", file=sys.stderr)
+ except Exception:
open_prs = 0
-
try:
closed_prs = gh_count_via_link_header(f"{GITHUB_API}/repos/{owner_repo}/pulls", {"state": "closed"})
- except Exception as e:
- if debug:
- print(f" · PR closed count fallback error for {owner_repo}: {e}", file=sys.stderr)
+ except Exception:
closed_prs = 0
-
- # Issue counts
open_issues = closed_issues = 0
try:
info = gh_repo_info(owner_repo)
- if not info.get("has_issues", True):
- if debug:
- print(f" · issues disabled for {owner_repo}; setting issue counts to 0")
- else:
+ if info.get("has_issues", True):
open_issues = gh_search_count_safe(f"repo:{owner_repo} is:issue is:open")
closed_issues = gh_search_count_safe(f"repo:{owner_repo} is:issue is:closed")
- except Exception as e:
- if debug:
- print(f" · issue count error for {owner_repo}: {e}", file=sys.stderr)
- # leave zeros
-
+ except Exception:
+ pass
return open_issues, closed_issues, open_prs, closed_prs
def gh_search_repos(query: str, per_page: int = 50) -> List[Dict[str, Any]]:
@@ -498,81 +527,38 @@ def load_repo_map(path: str) -> Dict[str, List[str]]:
mapping.setdefault(pid, []).append(repo)
return mapping
-def name_candidates(pod_id: str, meta: Dict[str, Any]) -> List[str]:
- cands = {norm(pod_id)}
- res = meta.get("resource") or meta.get("podling") or ""
- if res:
- cands.add(norm(res))
- for alias in (meta.get("resourceAliases") or []) + (meta.get("aliases") or []):
- if alias:
- cands.add(norm(alias))
- cands = {re.sub(r"^apache-", "", x) for x in cands}
- return sorted(cands)
-
-def discover_apache_repos(pod_id: str, meta: Dict[str, Any], debug: bool=False) -> List[str]:
- cands = name_candidates(pod_id, meta)
- if debug:
- print(f" · repo name candidates: {', '.join(cands)}")
+def name_candidates_from_xml(meta: Dict[str, Any]) -> List[str]:
+ cands = {norm(meta.get("id") or ""), norm(meta.get("name") or "")}
+ for a in meta.get("aliases", []) or []:
+ if a:
+ cands.add(norm(a))
+ cands.discard("")
+ return sorted({re.sub(r"^apache-", "", x) for x in cands})
+
+def discover_apache_repos_from_xml(meta: Dict[str, Any], debug: bool=False) -> List[str]:
+ cands = name_candidates_from_xml(meta)
found: List[str] = []
checked: Set[str] = set()
-
for base in cands:
for pat in (base, f"incubator-{base}"):
full = f"apache/{pat}"
if full in checked:
continue
checked.add(full)
- if gh_repo_exists_full(full):
- found.append(full)
-
- if found:
- return sorted(set(found))
-
- search_items: List[Dict[str, Any]] = []
- for base in cands:
- for q in (f"org:apache in:name {base}", f"org:apache in:name incubator-{base}"):
try:
- items = gh_search_repos(q, per_page=50)
- search_items.extend(items or [])
- except Exception as e:
- if debug:
- print(f" ! repo search failed for '{q}': {e}", file=sys.stderr)
- continue
-
- def keep(n: str) -> bool:
- n = n.lower()
- if any(n.endswith(suf) for suf in EXCLUDE_SUFFIXES):
- return False
- return any(n == x or n.startswith(f"{x}-") or n == f"incubator-{x}" or n.startswith(f"incubator-{x}-")
- for x in cands)
-
- for it in search_items:
- name = it.get("name", "")
- full = it.get("full_name", "")
- if not full.startswith("apache/"):
- continue
- if keep(name):
- found.append(full)
-
- # Dedup preserve order
- seen: Set[str] = set()
- rv: List[str] = []
- for f in found:
- if f not in seen:
- seen.add(f)
- rv.append(f)
-
- if debug and not rv:
- print(" · no candidate repos found in apache/", file=sys.stderr)
- return rv
+ http_json(f"{GITHUB_API}/repos/{full}")
+ found.append(full)
+ except Exception:
+ pass
+ return sorted(set(found))
# ---------------- Main ----------------
def main():
- parser = argparse.ArgumentParser(description="Generate <=6mo podling metrics CSV/MD (no template/scoring).")
+ parser = argparse.ArgumentParser(description="Generate 6mo podling metrics CSV/MD.")
parser.add_argument("--out", default="podlings_under6mo.csv", help="CSV output path")
parser.add_argument("--md", default="podlings_under6mo.md", help="Markdown table output path")
- parser.add_argument("--lists-activity", type=int, default=180, help="Days window for lists activity [default: 180]")
+ parser.add_argument("--lists-activity", type=int, default=180, help="Days window for lists/GitHub activity [default: 180]")
parser.add_argument("--repos", default="", help="CSV mapping podling->repo (columns: podling,repo)")
parser.add_argument("--emit-repo-map", default="", help="Write discovered/mapped repos to CSV")
parser.add_argument("--minutes-dir", default="", help="Path to board agenda text files (for mentor sign-offs & reports)")
@@ -581,44 +567,42 @@ def main():
parser.set_defaults(auto_repos=True)
parser.add_argument("--debug", action="store_true", help="Verbose debug logging")
parser.add_argument("--audit-pod", default="", help="Audit board_agenda matches for this pod (id or name) and exit")
-
args = parser.parse_args()
if (args.repos or args.auto_repos) and not GITHUB_TOKEN:
print("WARN: GitHub access without GITHUB_TOKEN may hit rate limits.", file=sys.stderr)
- print("Fetching podling metadata from Whimsy...")
- podlings = http_json(URL_PODLINGS).get("podling", {})
- pstatus = http_json(URL_PODLING_STATUS).get("podling", {})
+ if not args.minutes_dir:
+ print("NOTE: --minutes-dir not provided → report/sign-off columns will be 0.", file=sys.stderr)
+ else:
+ print(f"Using minutes_dir: {args.minutes_dir}")
+
+ print("Fetching podlings from podlings.xml...")
+ podlings_xml = load_podlings_xml(URL_PODLINGS_XML)
+
+ print("Fetching LDAP project membership (owners/members) from Whimsy...")
ldapproj = http_json(URL_LDAP_PROJECTS).get("projects", {})
# If auditing, run once and exit
if args.audit_pod:
+ lower = args.audit_pod.lower()
pid = None; pname = None
- for k, meta in podlings.items():
- if k.lower() == args.audit_pod.lower():
- pid = k; pname = meta.get("name", k); break
- if not pid:
- for k, meta in podlings.items():
- if str(meta.get("name","")).lower() == args.audit_pod.lower():
- pid = k; pname = meta.get("name", k); break
+ for rid, meta in podlings_xml.items():
+ if rid.lower() == lower or meta.get("name","").lower() == lower:
+ pid = rid; pname = meta.get("name", rid); break
if not pid:
pid = args.audit_pod; pname = args.audit_pod
print(f"Auditing pod: id='{pid}' name='{pname}' in {args.minutes_dir or '(no minutes_dir)'}")
- audit_minutes_for_pod(pid, pname, args.minutes_dir)
+ audit_minutes_for_pod(pid, pname, args.minutes_dir, args.lists_activity)
return
- # Optional explicit repo mappings
- repo_map: Dict[str, List[str]] = {}
- if args.repos:
- print(f"Loading repo mapping from {args.repos} ...")
- repo_map = load_repo_map(args.repos)
-
rows: List[Dict[str, Any]] = []
- per_pod_repos: Dict[str, List[str]] = {}
- for pod_id, meta in podlings.items():
- if pstatus.get(pod_id) != "current":
+ since_dt = datetime.now(timezone.utc) - timedelta(days=args.lists_activity)
+ until_dt = datetime.now(timezone.utc)
+
+ for rid, meta in podlings_xml.items():
+ if (meta.get("status") or "").lower() != "current":
continue
start = meta.get("startdate") or meta.get("start_date")
if not start:
@@ -627,89 +611,93 @@ def main():
sdate = dtp.parse(start).date()
except Exception:
continue
-
age_mo = months_between(sdate, TODAY)
- if age_mo > 6:
- continue
- if age_mo < 5:
+ if age_mo != 6:
continue
+ pod_id = rid
pod_name = meta.get("name", pod_id)
print(f"\nProcessing podling: {pod_name} (age {age_mo} months)")
proj = ldapproj.get(pod_id, {})
- owners = proj.get("owners", [])
- members = proj.get("members", [])
+ owners = proj.get("owners", []) or []
+ members = proj.get("members", []) or []
ppmc = len(owners)
committers = len(members)
- # Mentors
- mentors_field = meta.get("mentors") or meta.get("mentorsNames") or []
- if isinstance(mentors_field, dict):
- mentors_count = len(mentors_field)
- elif isinstance(mentors_field, list):
- mentors_count = len([m for m in mentors_field if m])
- else:
- mentors_count = 0
-
- # Mailing lists signals
- dev_unique, dev_release_votes, dev_emails, _ = lists_activity_dev(
- infer_dev_list_domain(pod_id, meta), args.lists_activity, args.debug
- )
- gen_release_votes = lists_release_votes_general(
- pod_id, pod_name, args.lists_activity, args.debug
- )
- new_ppmc_count, new_committers_count = lists_growth_general(
- pod_id, pod_name, args.lists_activity, args.debug
- )
-
- # Reports & mentor sign-offs from board agendas
+ mentors_count = len({
+ (m.get("username") or "").strip().lower() or (m.get("name") or "").strip().lower()
+ for m in (meta.get("mentors") or [])
+ if (m.get("name") or m.get("username"))
+ })
+
+ dev_domain = infer_dev_list_domain(pod_id)
+ dev_unique, dev_release_votes, dev_emails, _ = lists_activity_dev(dev_domain, args.lists_activity, args.debug)
+ gen_release_votes = lists_release_votes_general(pod_id, pod_name, args.lists_activity, args.debug)
+ new_ppmc_vote_threads, new_committer_vote_threads = lists_growth_general(pod_id, pod_name, args.lists_activity, args.debug)
+
reports_count = mentor_signoffs = 0
if args.minutes_dir:
try:
- reports_count, mentor_signoffs = count_reports_and_signoffs(pod_id, pod_name, args.minutes_dir, args.debug)
+ reports_count, mentor_signoffs = count_reports_and_signoffs(
+ pod_id, pod_name, args.minutes_dir, args.lists_activity, args.debug
+ )
except Exception as e:
print(f" ! agenda parse error for {pod_id}: {e}", file=sys.stderr)
- # GitHub repos
+ if args.debug:
+ print(f" SUMMARY for {pod_name}: mentors={mentors_count}, reports={reports_count}, signoffs={mentor_signoffs}")
+
repos: List[str] = []
- if pod_id.lower() in repo_map:
- repos = repo_map[pod_id.lower()]
- print(f" - Using mapped repos: {', '.join(repos)}")
- elif args.auto_repos:
- print(" - Auto-discovering GitHub repos under apache/ ...")
+ if args.auto_repos:
try:
- repos = discover_apache_repos(pod_id, meta, args.debug)
- except Exception as e:
- print(f" ! auto-discovery error: {e}", file=sys.stderr)
- if repos:
- print(f" · found: {', '.join(repos)}")
- else:
- print(" · none found (you can provide --repos to override)")
- else:
- print(" - Auto-discovery disabled and no explicit mapping provided.")
-
- per_pod_repos[pod_id] = repos[:]
+ repos = discover_apache_repos_from_xml(meta, args.debug)
+ except Exception:
+ repos = []
- # GitHub metrics
gh_contribs_set: Set[str] = set()
gh_open_prs = gh_closed_prs = gh_open_issues = gh_closed_issues = 0
if repos:
for repo in repos:
try:
- gh_contribs_set |= gh_repo_contributors(repo)
- except Exception as e:
- print(f" ! contributors error for {repo}: {e}", file=sys.stderr)
+ # very lightweight; safe to skip on errors
+ data = http_json(f"{GITHUB_API}/repos/{repo}/contributors", params={"anon": "1", "per_page": 100})
+ for p in (data or []):
+ if p.get("login"):
+ gh_contribs_set.add(p["login"].lower())
+ except Exception:
+ pass
try:
- o_i, c_i, o_p, c_p = gh_repo_issue_pr_breakdown(repo, args.debug)
+ o_i = gh_search_count_safe(f"repo:{repo} is:issue is:open")
+ c_i = gh_search_count_safe(f"repo:{repo} is:issue is:closed")
+ o_p = gh_count_via_link_header(f"{GITHUB_API}/repos/{repo}/pulls", {"state": "open"})
+ c_p = gh_count_via_link_header(f"{GITHUB_API}/repos/{repo}/pulls", {"state": "closed"})
gh_open_issues += o_i
gh_closed_issues+= c_i
gh_open_prs += o_p
gh_closed_prs += c_p
- except Exception as e:
- print(f" ! issue/PR count error for {repo}: {e}", file=sys.stderr)
- else:
- print(" - No GitHub repos mapped for this podling.")
+ except Exception:
+ pass
+
+ pr_merged_count = 0
+ if repos:
+ merged_since = since_dt.strftime("%Y-%m-%d")
+ merged_until = until_dt.strftime("%Y-%m-%d")
+ for repo in repos:
+ q = f'repo:{repo} is:pr is:merged merged:{merged_since}..{merged_until}'
+ try:
+ page = 1
+ while True:
+ data = http_json(f"{GITHUB_API}/search/issues", params={"q": q, "per_page": 100, "page": page})
+ items = (data or {}).get("items", [])
+ if not items:
+ break
+ pr_merged_count += len(items)
+ if len(items) < 100:
+ break
+ page += 1
+ except Exception:
+ pass
# Assemble output row
row = {
@@ -720,19 +708,16 @@ def main():
"ppmc_count": ppmc,
"committer_count": committers,
- # Mentor block
"mentors_count": mentors_count,
"reports_count": reports_count,
"mentor_signoffs": mentor_signoffs,
- # Lists activity
"unique_posters": dev_unique,
"dev_emails": dev_emails,
"dev_release_vote_threads": dev_release_votes,
"general_release_vote_threads": gen_release_votes,
"total_release_vote_threads": dev_release_votes + gen_release_votes,
- # GitHub
"github_contributors": len(gh_contribs_set) if repos else "",
"github_open_issues": gh_open_issues if repos else "",
"github_closed_issues": gh_closed_issues if repos else "",
@@ -740,14 +725,17 @@ def main():
"github_closed_prs": gh_closed_prs if repos else "",
"github_repos": ", ".join(repos) if repos else "",
- # Growth
- "new_ppmc": new_ppmc_count,
- "new_committers": new_committers_count,
+ "new_ppmc_vote_threads": new_ppmc_vote_threads,
+ "new_committer_vote_threads": new_committer_vote_threads,
+
+ "pr_merged_count": pr_merged_count if repos else "",
+ "merger_diversity": "",
+ "reviewer_diversity": "",
}
rows.append(row)
if not rows:
- print("\nNo podlings ≤ 6 months found.")
+ print("\nNo podlings 6 months old found.")
return
rows.sort(key=lambda r: (r["age_months"], r["start_date"], r["podling"]))
---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org