You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/07/09 01:27:23 UTC
(tooling-agents) branch main updated: Fix bernies maintainer accuracy, add Go/Maven/RubyGems checkers
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-agents.git
The following commit(s) were added to refs/heads/main by this push:
new 2e78294 Fix bernies maintainer accuracy, add Go/Maven/RubyGems checkers
2e78294 is described below
commit 2e7829477ee809cd62b620759091eee41a8fdd94
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Wed Jul 8 18:27:14 2026 -0700
Fix bernies maintainer accuracy, add Go/Maven/RubyGems checkers
PyPI fix: parse maintainer_email and author_email fields (comma-
separated Name <email> format) instead of the free-text author/
maintainer fields which are usually empty. When email parsing
yields ≤1 person and a GitHub URL is available in project_urls,
falls back to GitHub contributors API. Fixes requests showing
1 maintainer instead of 3, pydantic showing 1 instead of 12, etc.
New ecosystem checkers:
- RubyGems: /owners.json endpoint, falls back to GitHub via
source_code_uri
- Go: module path → GitHub repo (github.com/* paths direct,
vanity URLs for golang.org/x, go.uber.org, google.golang.org)
- Maven: POM from Maven Central, extracts GitHub URL from <scm>
and <url> tags, falls back to apache/{project} guess for
org.apache.* groupIds
All six checkers share get_github_maintainers() helper for
consistent GitHub fallback. Previously only PyPI/npm/crates.io
were checked (334+533+413 deps), leaving Maven (1407) and Go
(857) with zero maintainer data.
---
gha-review/agents/gha_bernies.py | 284 +++++++++++++++++++++++++++++++++++++--
1 file changed, 273 insertions(+), 11 deletions(-)
diff --git a/gha-review/agents/gha_bernies.py b/gha-review/agents/gha_bernies.py
index 5de6476..45f2d94 100644
--- a/gha-review/agents/gha_bernies.py
+++ b/gha-review/agents/gha_bernies.py
@@ -190,6 +190,41 @@ async def run(input_dict, tools):
# ── Registry API queries for maintainer data ──
+ async def extract_github_url(info):
+ """Extract GitHub repo URL from PyPI project_urls."""
+ urls = info.get('project_urls') or {}
+ for key in ('Source', 'source', 'Source Code', 'Repository', 'GitHub', 'Code'):
+ url = urls.get(key, '')
+ if url and 'github.com' in url.lower():
+ # Normalize to owner/repo
+ m = re.search(r'github\.com/([^/]+/[^/]+)', url)
+ if m:
+ return m.group(1).rstrip('/')
+ # Fallback: check home_page
+ hp = info.get('home_page', '') or ''
+ if 'github.com' in hp.lower():
+ m = re.search(r'github\.com/([^/]+/[^/]+)', hp)
+ if m:
+ return m.group(1).rstrip('/')
+ return None
+
+ def parse_email_list(field):
+ """Extract names from comma-separated 'Name <email>' fields."""
+ if not field:
+ return set()
+ people = set()
+ for entry in field.split(','):
+ entry = entry.strip()
+ if not entry:
+ continue
+ # Extract name from "Name <email>" format
+ m = re.match(r'([^<]+?)(?:\s*<[^>]+>)?$', entry)
+ if m:
+ name = m.group(1).strip()
+ if name and name.lower() not in ('none', '', 'unknown'):
+ people.add(name.lower())
+ return people
+
async def check_pypi(pkg):
try:
r = await http_client.get(f"https://pypi.org/pypi/{pkg}/json")
@@ -197,25 +232,53 @@ async def run(input_dict, tools):
return None
data = r.json()
info = data.get('info', {})
- maintainers = info.get('maintainer', '') or ''
- author = info.get('author', '') or ''
+
+ # Parse all email fields for maintainer names
maintainer_email = info.get('maintainer_email', '') or ''
author_email = info.get('author_email', '') or ''
- # Count distinct people
+ maintainer = info.get('maintainer', '') or ''
+ author = info.get('author', '') or ''
+
people = set()
- for field in [maintainers, author]:
- for name in field.split(','):
- name = name.strip()
- if name and name.lower() not in ('none', ''):
- people.add(name.lower())
+ people.update(parse_email_list(maintainer_email))
+ people.update(parse_email_list(author_email))
+ # Fall back to free-text fields if emails didn't yield names
+ if not people:
+ for field in [maintainer, author]:
+ for name in field.split(','):
+ name = name.strip()
+ if name and name.lower() not in ('none', ''):
+ people.add(name.lower())
+
+ # Extract GitHub URL for source_repo
+ gh_repo = await extract_github_url(info)
+ source_url = f"https://github.com/{gh_repo}" if gh_repo else None
+
+ # If ≤1 person found and GitHub URL available, check GitHub contributors
+ if len(people) <= 1 and gh_repo:
+ try:
+ gh_resp = await http_client.get(
+ f"https://api.github.com/repos/{gh_repo}/contributors?per_page=10&anon=false",
+ headers=github_headers)
+ if gh_resp.status_code == 200:
+ contribs = gh_resp.json()
+ if isinstance(contribs, list) and len(contribs) > len(people):
+ # Use GitHub contributor count instead
+ gh_names = [c.get('login', '?') for c in contribs[:5]
+ if not c.get('login', '').endswith('[bot]')]
+ if len(gh_names) > len(people):
+ people = set(gh_names)
+ except Exception:
+ pass
+
return {
'name': pkg,
'ecosystem': 'pypi',
'version': info.get('version', '?'),
'maintainer_count': len(people) if people else 1,
- 'maintainers': list(people)[:5],
- 'home_page': info.get('home_page', '') or info.get('project_url', ''),
- 'source_repo': None,
+ 'maintainers': sorted(list(people))[:5],
+ 'home_page': info.get('home_page', '') or '',
+ 'source_repo': source_url,
}
except Exception:
return None
@@ -274,6 +337,205 @@ async def run(input_dict, tools):
'crates_io': check_crates,
}
+ # ── GitHub contributor helper (shared by all checkers) ──
+
+ async def get_github_maintainers(gh_repo):
+ """Query GitHub contributors API. Returns (count, [names])."""
+ try:
+ resp = await http_client.get(
+ f"https://api.github.com/repos/{gh_repo}/contributors?per_page=30&anon=false",
+ headers=github_headers)
+ if resp.status_code != 200:
+ return 0, []
+ contribs = resp.json()
+ if not isinstance(contribs, list):
+ return 0, []
+ # Filter bots, get human contributors
+ humans = [c for c in contribs if not c.get('login', '').endswith('[bot]')]
+ names = [c.get('login', '?') for c in humans[:5]]
+ return len(humans), names
+ except Exception:
+ return 0, []
+
+ # ── RubyGems checker ──
+
+ async def check_rubygems(pkg):
+ try:
+ # Get gem info (public, no auth needed)
+ r = await http_client.get(f"https://rubygems.org/api/v1/gems/{pkg}.json")
+ if r.status_code != 200:
+ return None
+ data = r.json()
+ version = data.get('version', '?')
+ source_url = data.get('source_code_uri', '') or data.get('homepage_uri', '')
+
+ # Try owners endpoint (may require auth)
+ owners_count = 0
+ owner_names = []
+ try:
+ owners_resp = await http_client.get(
+ f"https://rubygems.org/api/v1/gems/{pkg}/owners.json")
+ if owners_resp.status_code == 200:
+ owners = owners_resp.json()
+ if isinstance(owners, list):
+ owners_count = len(owners)
+ owner_names = [o.get('handle', '?') for o in owners[:5]]
+ except Exception:
+ pass
+
+ # If owners API failed or returned few, try GitHub
+ gh_repo = None
+ if source_url and 'github.com' in source_url.lower():
+ m = re.search(r'github\.com/([^/]+/[^/]+)', source_url)
+ if m:
+ gh_repo = m.group(1).rstrip('/')
+
+ if owners_count <= 1 and gh_repo:
+ gh_count, gh_names = await get_github_maintainers(gh_repo)
+ if gh_count > owners_count:
+ owners_count = gh_count
+ owner_names = gh_names
+
+ return {
+ 'name': pkg,
+ 'ecosystem': 'rubygems',
+ 'version': version,
+ 'maintainer_count': owners_count if owners_count > 0 else 1,
+ 'maintainers': owner_names,
+ 'source_repo': f"https://github.com/{gh_repo}" if gh_repo else source_url,
+ }
+ except Exception:
+ return None
+
+ # ── Go module checker ──
+
+ async def check_go(pkg):
+ try:
+ # Go module paths that start with github.com/ are direct repos
+ gh_repo = None
+ if pkg.startswith('github.com/'):
+ parts = pkg.replace('github.com/', '').split('/')
+ if len(parts) >= 2:
+ gh_repo = f"{parts[0]}/{parts[1]}"
+
+ # For non-github paths, try to resolve via go-import meta tag
+ if not gh_repo:
+ # Common Go vanity URL patterns
+ vanity_map = {
+ 'golang.org/x/': 'golang/',
+ 'go.uber.org/': 'uber-go/',
+ 'google.golang.org/': 'google/', # usually googleapis or grpc
+ }
+ for prefix, gh_org in vanity_map.items():
+ if pkg.startswith(prefix):
+ mod_name = pkg.replace(prefix, '').split('/')[0]
+ gh_repo = f"{gh_org}{mod_name}"
+ break
+
+ if not gh_repo:
+ return None # Can't resolve to GitHub
+
+ # Check if repo exists and get contributor data
+ repo_resp = await http_client.get(
+ f"https://api.github.com/repos/{gh_repo}",
+ headers=github_headers)
+ if repo_resp.status_code != 200:
+ return None
+
+ count, names = await get_github_maintainers(gh_repo)
+
+ return {
+ 'name': pkg,
+ 'ecosystem': 'go',
+ 'version': '?',
+ 'maintainer_count': count if count > 0 else 1,
+ 'maintainers': names,
+ 'source_repo': f"https://github.com/{gh_repo}",
+ }
+ except Exception:
+ return None
+
+ # ── Maven checker ──
+
+ async def check_maven(pkg):
+ try:
+ if ':' not in pkg:
+ return None
+ group_id, artifact_id = pkg.split(':', 1)
+
+ # Search Maven Central for the artifact
+ search_url = (f"https://search.maven.org/solrsearch/select?"
+ f"q=g:{group_id}+AND+a:{artifact_id}&rows=1&wt=json")
+ r = await http_client.get(search_url)
+ if r.status_code != 200:
+ return None
+ data = r.json()
+ docs = data.get('response', {}).get('docs', [])
+ if not docs:
+ return None
+ doc = docs[0]
+ version = doc.get('latestVersion', '?')
+
+ # Try to get POM for GitHub URL
+ group_path = group_id.replace('.', '/')
+ pom_url = (f"https://repo1.maven.org/maven2/{group_path}/"
+ f"{artifact_id}/{version}/{artifact_id}-{version}.pom")
+ gh_repo = None
+ try:
+ pom_resp = await http_client.get(pom_url, timeout=10)
+ if pom_resp.status_code == 200:
+ pom = pom_resp.text
+ # Look for GitHub URL in POM
+ for pattern in [
+ r'<url>\s*(https?://github\.com/[^<]+)\s*</url>',
+ r'<connection>\s*scm:git:(?:https?://)?(?:git@)?github\.com[:/]([^<]+?)(?:\.git)?\s*</connection>',
+ r'<developerConnection>[^<]*github\.com[:/]([^<]+?)(?:\.git)?</developerConnection>',
+ ]:
+ m = re.search(pattern, pom, re.IGNORECASE)
+ if m:
+ url_or_path = m.group(1)
+ # Extract owner/repo
+ m2 = re.search(r'github\.com/([^/]+/[^/\s<]+)', url_or_path)
+ if m2:
+ gh_repo = m2.group(1).rstrip('/')
+ elif '/' in url_or_path and not url_or_path.startswith('http'):
+ gh_repo = url_or_path.rstrip('/')
+ if gh_repo:
+ break
+ except Exception:
+ pass
+
+ if not gh_repo:
+ # Guess: many Apache projects follow apache/{artifactId} pattern
+ if group_id.startswith('org.apache.'):
+ project = group_id.replace('org.apache.', '').split('.')[0]
+ gh_repo = f"apache/{project}"
+
+ if not gh_repo:
+ return None
+
+ count, names = await get_github_maintainers(gh_repo)
+
+ return {
+ 'name': pkg,
+ 'ecosystem': 'maven',
+ 'version': version,
+ 'maintainer_count': count if count > 0 else 1,
+ 'maintainers': names,
+ 'source_repo': f"https://github.com/{gh_repo}",
+ }
+ except Exception:
+ return None
+
+ REGISTRY_CHECKS = {
+ 'pypi': check_pypi,
+ 'npm': check_npm,
+ 'crates_io': check_crates,
+ 'rubygems': check_rubygems,
+ 'go': check_go,
+ 'maven': check_maven,
+ }
+
# ── Discover repos from prefetch ──
wf_ns = data_store.use_namespace(f"ci-workflows:{github_owner}")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org