You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/07/09 01:27:23 UTC

(tooling-agents) branch main updated: Fix bernies maintainer accuracy, add Go/Maven/RubyGems checkers

This is an automated email from the ASF dual-hosted git repository.

andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-agents.git


The following commit(s) were added to refs/heads/main by this push:
     new 2e78294  Fix bernies maintainer accuracy, add Go/Maven/RubyGems checkers
2e78294 is described below

commit 2e7829477ee809cd62b620759091eee41a8fdd94
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Wed Jul 8 18:27:14 2026 -0700

    Fix bernies maintainer accuracy, add Go/Maven/RubyGems checkers
    
    PyPI fix: parse maintainer_email and author_email fields (comma-
    separated Name <email> format) instead of the free-text author/
    maintainer fields which are usually empty. When email parsing
    yields ≤1 person and a GitHub URL is available in project_urls,
    falls back to GitHub contributors API. Fixes requests showing
    1 maintainer instead of 3, pydantic showing 1 instead of 12, etc.
    
    New ecosystem checkers:
    - RubyGems: /owners.json endpoint, falls back to GitHub via
      source_code_uri
    - Go: module path → GitHub repo (github.com/* paths direct,
      vanity URLs for golang.org/x, go.uber.org, google.golang.org)
    - Maven: POM from Maven Central, extracts GitHub URL from <scm>
      and <url> tags, falls back to apache/{project} guess for
      org.apache.* groupIds
    
    All six checkers share get_github_maintainers() helper for
    consistent GitHub fallback. Previously only PyPI/npm/crates.io
    were checked (334+533+413 deps), leaving Maven (1407) and Go
    (857) with zero maintainer data.
---
 gha-review/agents/gha_bernies.py | 284 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 273 insertions(+), 11 deletions(-)

diff --git a/gha-review/agents/gha_bernies.py b/gha-review/agents/gha_bernies.py
index 5de6476..45f2d94 100644
--- a/gha-review/agents/gha_bernies.py
+++ b/gha-review/agents/gha_bernies.py
@@ -190,6 +190,41 @@ async def run(input_dict, tools):
 
         # ── Registry API queries for maintainer data ──
 
+        async def extract_github_url(info):
+            """Extract GitHub repo URL from PyPI project_urls."""
+            urls = info.get('project_urls') or {}
+            for key in ('Source', 'source', 'Source Code', 'Repository', 'GitHub', 'Code'):
+                url = urls.get(key, '')
+                if url and 'github.com' in url.lower():
+                    # Normalize to owner/repo
+                    m = re.search(r'github\.com/([^/]+/[^/]+)', url)
+                    if m:
+                        return m.group(1).rstrip('/')
+            # Fallback: check home_page
+            hp = info.get('home_page', '') or ''
+            if 'github.com' in hp.lower():
+                m = re.search(r'github\.com/([^/]+/[^/]+)', hp)
+                if m:
+                    return m.group(1).rstrip('/')
+            return None
+
+        def parse_email_list(field):
+            """Extract names from comma-separated 'Name <email>' fields."""
+            if not field:
+                return set()
+            people = set()
+            for entry in field.split(','):
+                entry = entry.strip()
+                if not entry:
+                    continue
+                # Extract name from "Name <email>" format
+                m = re.match(r'([^<]+?)(?:\s*<[^>]+>)?$', entry)
+                if m:
+                    name = m.group(1).strip()
+                    if name and name.lower() not in ('none', '', 'unknown'):
+                        people.add(name.lower())
+            return people
+
         async def check_pypi(pkg):
             try:
                 r = await http_client.get(f"https://pypi.org/pypi/{pkg}/json")
@@ -197,25 +232,53 @@ async def run(input_dict, tools):
                     return None
                 data = r.json()
                 info = data.get('info', {})
-                maintainers = info.get('maintainer', '') or ''
-                author = info.get('author', '') or ''
+
+                # Parse all email fields for maintainer names
                 maintainer_email = info.get('maintainer_email', '') or ''
                 author_email = info.get('author_email', '') or ''
-                # Count distinct people
+                maintainer = info.get('maintainer', '') or ''
+                author = info.get('author', '') or ''
+
                 people = set()
-                for field in [maintainers, author]:
-                    for name in field.split(','):
-                        name = name.strip()
-                        if name and name.lower() not in ('none', ''):
-                            people.add(name.lower())
+                people.update(parse_email_list(maintainer_email))
+                people.update(parse_email_list(author_email))
+                # Fall back to free-text fields if emails didn't yield names
+                if not people:
+                    for field in [maintainer, author]:
+                        for name in field.split(','):
+                            name = name.strip()
+                            if name and name.lower() not in ('none', ''):
+                                people.add(name.lower())
+
+                # Extract GitHub URL for source_repo
+                gh_repo = await extract_github_url(info)
+                source_url = f"https://github.com/{gh_repo}" if gh_repo else None
+
+                # If ≤1 person found and GitHub URL available, check GitHub contributors
+                if len(people) <= 1 and gh_repo:
+                    try:
+                        gh_resp = await http_client.get(
+                            f"https://api.github.com/repos/{gh_repo}/contributors?per_page=10&anon=false",
+                            headers=github_headers)
+                        if gh_resp.status_code == 200:
+                            contribs = gh_resp.json()
+                            if isinstance(contribs, list) and len(contribs) > len(people):
+                                # Use GitHub contributor count instead
+                                gh_names = [c.get('login', '?') for c in contribs[:5]
+                                           if not c.get('login', '').endswith('[bot]')]
+                                if len(gh_names) > len(people):
+                                    people = set(gh_names)
+                    except Exception:
+                        pass
+
                 return {
                     'name': pkg,
                     'ecosystem': 'pypi',
                     'version': info.get('version', '?'),
                     'maintainer_count': len(people) if people else 1,
-                    'maintainers': list(people)[:5],
-                    'home_page': info.get('home_page', '') or info.get('project_url', ''),
-                    'source_repo': None,
+                    'maintainers': sorted(list(people))[:5],
+                    'home_page': info.get('home_page', '') or '',
+                    'source_repo': source_url,
                 }
             except Exception:
                 return None
@@ -274,6 +337,205 @@ async def run(input_dict, tools):
             'crates_io': check_crates,
         }
 
+        # ── GitHub contributor helper (shared by all checkers) ──
+
+        async def get_github_maintainers(gh_repo):
+            """Query GitHub contributors API. Returns (count, [names])."""
+            try:
+                resp = await http_client.get(
+                    f"https://api.github.com/repos/{gh_repo}/contributors?per_page=30&anon=false",
+                    headers=github_headers)
+                if resp.status_code != 200:
+                    return 0, []
+                contribs = resp.json()
+                if not isinstance(contribs, list):
+                    return 0, []
+                # Filter bots, get human contributors
+                humans = [c for c in contribs if not c.get('login', '').endswith('[bot]')]
+                names = [c.get('login', '?') for c in humans[:5]]
+                return len(humans), names
+            except Exception:
+                return 0, []
+
+        # ── RubyGems checker ──
+
+        async def check_rubygems(pkg):
+            try:
+                # Get gem info (public, no auth needed)
+                r = await http_client.get(f"https://rubygems.org/api/v1/gems/{pkg}.json")
+                if r.status_code != 200:
+                    return None
+                data = r.json()
+                version = data.get('version', '?')
+                source_url = data.get('source_code_uri', '') or data.get('homepage_uri', '')
+
+                # Try owners endpoint (may require auth)
+                owners_count = 0
+                owner_names = []
+                try:
+                    owners_resp = await http_client.get(
+                        f"https://rubygems.org/api/v1/gems/{pkg}/owners.json")
+                    if owners_resp.status_code == 200:
+                        owners = owners_resp.json()
+                        if isinstance(owners, list):
+                            owners_count = len(owners)
+                            owner_names = [o.get('handle', '?') for o in owners[:5]]
+                except Exception:
+                    pass
+
+                # If owners API failed or returned few, try GitHub
+                gh_repo = None
+                if source_url and 'github.com' in source_url.lower():
+                    m = re.search(r'github\.com/([^/]+/[^/]+)', source_url)
+                    if m:
+                        gh_repo = m.group(1).rstrip('/')
+
+                if owners_count <= 1 and gh_repo:
+                    gh_count, gh_names = await get_github_maintainers(gh_repo)
+                    if gh_count > owners_count:
+                        owners_count = gh_count
+                        owner_names = gh_names
+
+                return {
+                    'name': pkg,
+                    'ecosystem': 'rubygems',
+                    'version': version,
+                    'maintainer_count': owners_count if owners_count > 0 else 1,
+                    'maintainers': owner_names,
+                    'source_repo': f"https://github.com/{gh_repo}" if gh_repo else source_url,
+                }
+            except Exception:
+                return None
+
+        # ── Go module checker ──
+
+        async def check_go(pkg):
+            try:
+                # Go module paths that start with github.com/ are direct repos
+                gh_repo = None
+                if pkg.startswith('github.com/'):
+                    parts = pkg.replace('github.com/', '').split('/')
+                    if len(parts) >= 2:
+                        gh_repo = f"{parts[0]}/{parts[1]}"
+
+                # For non-github paths, try to resolve via go-import meta tag
+                if not gh_repo:
+                    # Common Go vanity URL patterns
+                    vanity_map = {
+                        'golang.org/x/': 'golang/',
+                        'go.uber.org/': 'uber-go/',
+                        'google.golang.org/': 'google/',  # usually googleapis or grpc
+                    }
+                    for prefix, gh_org in vanity_map.items():
+                        if pkg.startswith(prefix):
+                            mod_name = pkg.replace(prefix, '').split('/')[0]
+                            gh_repo = f"{gh_org}{mod_name}"
+                            break
+
+                if not gh_repo:
+                    return None  # Can't resolve to GitHub
+
+                # Check if repo exists and get contributor data
+                repo_resp = await http_client.get(
+                    f"https://api.github.com/repos/{gh_repo}",
+                    headers=github_headers)
+                if repo_resp.status_code != 200:
+                    return None
+
+                count, names = await get_github_maintainers(gh_repo)
+
+                return {
+                    'name': pkg,
+                    'ecosystem': 'go',
+                    'version': '?',
+                    'maintainer_count': count if count > 0 else 1,
+                    'maintainers': names,
+                    'source_repo': f"https://github.com/{gh_repo}",
+                }
+            except Exception:
+                return None
+
+        # ── Maven checker ──
+
+        async def check_maven(pkg):
+            try:
+                if ':' not in pkg:
+                    return None
+                group_id, artifact_id = pkg.split(':', 1)
+
+                # Search Maven Central for the artifact
+                search_url = (f"https://search.maven.org/solrsearch/select?"
+                             f"q=g:{group_id}+AND+a:{artifact_id}&rows=1&wt=json")
+                r = await http_client.get(search_url)
+                if r.status_code != 200:
+                    return None
+                data = r.json()
+                docs = data.get('response', {}).get('docs', [])
+                if not docs:
+                    return None
+                doc = docs[0]
+                version = doc.get('latestVersion', '?')
+
+                # Try to get POM for GitHub URL
+                group_path = group_id.replace('.', '/')
+                pom_url = (f"https://repo1.maven.org/maven2/{group_path}/"
+                          f"{artifact_id}/{version}/{artifact_id}-{version}.pom")
+                gh_repo = None
+                try:
+                    pom_resp = await http_client.get(pom_url, timeout=10)
+                    if pom_resp.status_code == 200:
+                        pom = pom_resp.text
+                        # Look for GitHub URL in POM
+                        for pattern in [
+                            r'<url>\s*(https?://github\.com/[^<]+)\s*</url>',
+                            r'<connection>\s*scm:git:(?:https?://)?(?:git@)?github\.com[:/]([^<]+?)(?:\.git)?\s*</connection>',
+                            r'<developerConnection>[^<]*github\.com[:/]([^<]+?)(?:\.git)?</developerConnection>',
+                        ]:
+                            m = re.search(pattern, pom, re.IGNORECASE)
+                            if m:
+                                url_or_path = m.group(1)
+                                # Extract owner/repo
+                                m2 = re.search(r'github\.com/([^/]+/[^/\s<]+)', url_or_path)
+                                if m2:
+                                    gh_repo = m2.group(1).rstrip('/')
+                                elif '/' in url_or_path and not url_or_path.startswith('http'):
+                                    gh_repo = url_or_path.rstrip('/')
+                                if gh_repo:
+                                    break
+                except Exception:
+                    pass
+
+                if not gh_repo:
+                    # Guess: many Apache projects follow apache/{artifactId} pattern
+                    if group_id.startswith('org.apache.'):
+                        project = group_id.replace('org.apache.', '').split('.')[0]
+                        gh_repo = f"apache/{project}"
+
+                if not gh_repo:
+                    return None
+
+                count, names = await get_github_maintainers(gh_repo)
+
+                return {
+                    'name': pkg,
+                    'ecosystem': 'maven',
+                    'version': version,
+                    'maintainer_count': count if count > 0 else 1,
+                    'maintainers': names,
+                    'source_repo': f"https://github.com/{gh_repo}",
+                }
+            except Exception:
+                return None
+
+        REGISTRY_CHECKS = {
+            'pypi': check_pypi,
+            'npm': check_npm,
+            'crates_io': check_crates,
+            'rubygems': check_rubygems,
+            'go': check_go,
+            'maven': check_maven,
+        }
+
         # ── Discover repos from prefetch ──
 
         wf_ns = data_store.use_namespace(f"ci-workflows:{github_owner}")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org