You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@incubator.apache.org by jm...@apache.org on 2026/01/24 19:53:06 UTC

(incubator) 28/30: fix young projects

This is an automated email from the ASF dual-hosted git repository.

jmclean pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator.git

commit 53b05ec6c29172cfa1f9d3deb7e5612bae2e0e07
Author: Justin Mclean <jm...@apache.org>
AuthorDate: Sat Jan 24 20:59:53 2026 +1100

    fix young projects
---
 app.py | 288 +++++++++++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 200 insertions(+), 88 deletions(-)

diff --git a/app.py b/app.py
index 3bf6c70d6..f3e4f5558 100644
--- a/app.py
+++ b/app.py
@@ -10,9 +10,8 @@ import pandas as pd
 import streamlit as st
 import altair as alt
 
-
 # ============================
-# Git helpers (ALWAYS use ./incubator)
+# Git helpers (use THIS repo)
 # ============================
 
 def run_git(repo_dir: Path, args: list[str]) -> str:
@@ -29,13 +28,12 @@ def run_git(repo_dir: Path, args: list[str]) -> str:
     return p.stdout
 
 
-def resolve_incubator_repo() -> Path:
+def resolve_repo_root() -> Path:
     """
     Use the dashboard repo itself as the git root.
     This works locally and on Streamlit Community Cloud.
     """
     cwd = Path.cwd().resolve()
-
     try:
         p = subprocess.run(
             ["git", "-C", str(cwd), "rev-parse", "--show-toplevel"],
@@ -212,7 +210,7 @@ def parse_window_details_block(bullet_lines: list[str]) -> dict:
     return row
 
 
-def parse_report(text: str, podling: str) -> list[dict]:
+def parse_report(text: str, podling_stem: str) -> list[dict]:
     m = RE_GENERATED_ON.search(text)
     if not m:
         return []
@@ -235,18 +233,21 @@ def parse_report(text: str, podling: str) -> list[dict]:
         bullet_lines = [ln for ln in block.splitlines() if ln.strip().startswith("- ")]
         metrics = parse_window_details_block(bullet_lines)
 
-        out.append({
-            "podling": podling,
-            "snapshot_date": snapshot_date,
-            "window": window,
-            **metrics,
-        })
+        out.append(
+            {
+                "podling_key": podling_stem.casefold(),  # merge Hamalton/hamalton/etc
+                "podling_display": podling_stem,         # keep original for display; latest wins
+                "snapshot_date": snapshot_date,
+                "window": window,
+                **metrics,
+            }
+        )
 
     return out
 
 
 # ============================
-# Dataset build (git history from incubator repo)
+# Dataset build (git history)
 # ============================
 
 @st.cache_data(show_spinner=True)
@@ -267,8 +268,9 @@ def build_dataset(repo_root: str, reports_dir: str, max_commits: int) -> pd.Data
             text = blob_text.get(oid)
             if not text:
                 continue
-            podling = Path(path).stem
-            for r in parse_report(text, podling):
+
+            podling_stem = Path(path).stem
+            for r in parse_report(text, podling_stem):
                 r["commit"] = commit
                 r["path"] = path
                 rows.append(r)
@@ -280,29 +282,61 @@ def build_dataset(repo_root: str, reports_dir: str, max_commits: int) -> pd.Data
     df["snapshot_date"] = pd.to_datetime(df["snapshot_date"], errors="coerce")
     df = df.dropna(subset=["snapshot_date"])
 
-    df = df.sort_values(["podling", "window", "snapshot_date", "commit"])
-    df = df.drop_duplicates(["podling", "window", "snapshot_date"], keep="last")
+    # de-dupe
+    df = df.sort_values(["podling_key", "window", "snapshot_date", "commit"])
+    df = df.drop_duplicates(["podling_key", "window", "snapshot_date"], keep="last")
+
+    # ensure one display name per key: latest snapshot wins
+    latest_name = (
+        df.sort_values(["podling_key", "snapshot_date", "commit"])
+        .groupby("podling_key", as_index=False)
+        .tail(1)[["podling_key", "podling_display"]]
+    )
+    df = df.drop(columns=["podling_display"]).merge(latest_name, on="podling_key", how="left")
+
     return df.reset_index(drop=True)
 
 
-def combined_snapshot(df: pd.DataFrame) -> pd.DataFrame:
-    d = df.sort_values(["podling", "window", "snapshot_date"])
-    latest = d.groupby(["podling", "window"], as_index=False).tail(1)
+# ============================
+# Latest-only combined snapshot (STRICT)
+# - availability and signals come ONLY from the latest snapshot per podling_key
+# - windows count ONLY if present in that latest snapshot (no inference)
+# ============================
 
-    metrics = [c for c in latest.columns if c not in ("podling", "window", "snapshot_date", "commit", "path")]
-    wide = latest.pivot(index="podling", columns="window", values=metrics)
+def latest_combo(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty:
+        return df
+
+    # latest snapshot_date per podling_key
+    latest_dates = df.groupby("podling_key", as_index=False)["snapshot_date"].max()
+    latest = df.merge(latest_dates, on=["podling_key", "snapshot_date"], how="inner")
+
+    # if multiple commits same day, keep latest commit row per (key, window)
+    latest = latest.sort_values(["podling_key", "window", "snapshot_date", "commit"]).groupby(
+        ["podling_key", "window"], as_index=False
+    ).tail(1)
+
+    keep = ["podling_key", "podling_display", "window", "snapshot_date"]
+    metrics = [c for c in latest.columns if c not in (keep + ["commit", "path"])]
+
+    wide = latest.pivot(index="podling_key", columns="window", values=metrics)
     wide.columns = [f"{m}_{w}" for (m, w) in wide.columns]
     wide = wide.reset_index()
 
-    dates = latest.pivot(index="podling", columns="window", values="snapshot_date")
+    dates = latest.pivot(index="podling_key", columns="window", values="snapshot_date")
     dates.columns = [f"snapshot_date_{w}" for w in dates.columns]
     dates = dates.reset_index()
 
-    return wide.merge(dates, on="podling", how="left")
+    names = latest.groupby("podling_key", as_index=False).tail(1)[["podling_key", "podling_display"]]
+    out = wide.merge(dates, on="podling_key", how="left").merge(names, on="podling_key", how="left")
+
+    # add single "latest_snapshot_date" (for info / debugging if needed)
+    out["latest_snapshot_date"] = latest.groupby("podling_key")["snapshot_date"].max().reindex(out["podling_key"]).values
+    return out
 
 
 # ============================
-# Signals + commentary (copyable)
+# Signals + commentary (STRICT latest-only windows)
 # ============================
 
 PRIMARY = "primary"
@@ -321,39 +355,59 @@ def _val(row: pd.Series, key: str) -> Optional[float]:
     return None
 
 
-def build_signals_combined(row: pd.Series) -> list[dict]:
+def _has_window(row: pd.Series, w: str) -> bool:
+    d = row.get(f"snapshot_date_{w}")
+    return d is not None and not pd.isna(d)
+
+
+def window_availability_messages(row: pd.Series) -> list[str]:
+    # STRICT: only based on latest snapshot (presence of ### window section in that latest file)
+    msgs: list[str] = []
+    for w in ["12m", "6m", "3m"]:
+        msgs.append(f"- **{w}**: {'available' if _has_window(row, w) else 'not available'}")
+    return msgs
+
+
+def build_signals_latest_only(row: pd.Series) -> list[dict]:
     sig: list[dict] = []
 
     def add(level: str, text: str):
         sig.append({"level": level, "text": text})
 
-    r6 = _val(row, "releases_6m")
-    r12 = _val(row, "releases_12m")
+    has3 = _has_window(row, "3m")
+    has6 = _has_window(row, "6m")
+    has12 = _has_window(row, "12m")
 
-    p3 = _val(row, "dev_unique_posters_3m")
-    p12 = _val(row, "dev_unique_posters_12m")
+    # pull values ONLY if the window exists
+    r6 = _val(row, "releases_6m") if has6 else None
+    r12 = _val(row, "releases_12m") if has12 else None
 
-    m3 = _val(row, "dev_msgs_3m")
-    m12 = _val(row, "dev_msgs_12m")
+    p3 = _val(row, "dev_unique_posters_3m") if has3 else None
+    p12 = _val(row, "dev_unique_posters_12m") if has12 else None
 
-    bus3 = _val(row, "bus50_3m")
-    committers12 = _val(row, "unique_committers_12m")
+    m3 = _val(row, "dev_msgs_3m") if has3 else None
+    m12 = _val(row, "dev_msgs_12m") if has12 else None
 
-    merge3 = _val(row, "median_merge_days_3m")
-    merge12 = _val(row, "median_merge_days_12m")
+    bus3 = _val(row, "bus50_3m") if has3 else None
+    committers12 = _val(row, "unique_committers_12m") if has12 else None
 
-    prs3 = _val(row, "prs_merged_3m")
-    revdiv3 = _val(row, "reviewer_div_eff_3m")
-    authdiv3 = _val(row, "pr_author_div_eff_3m")
+    merge3 = _val(row, "median_merge_days_3m") if has3 else None
+    merge12 = _val(row, "median_merge_days_12m") if has12 else None
 
-    commits3 = _val(row, "commits_3m")
+    prs3 = _val(row, "prs_merged_3m") if has3 else None
+    revdiv3 = _val(row, "reviewer_div_eff_3m") if has3 else None
+    authdiv3 = _val(row, "pr_author_div_eff_3m") if has3 else None
 
-    if r12 is not None and r12 == 0:
+    commits3 = _val(row, "commits_3m") if has3 else None
+
+    # Releases checks: only when the relevant windows exist in the LATEST report
+    if has12 and r12 is not None and r12 == 0:
         add(PRIMARY, "No releases in 12m — likely needs mentor/IPMC attention (blockers/ownership/process).")
-    elif r6 is not None and r6 == 0 and (r12 is not None and r12 > 0):
+    elif has6 and has12 and r6 is not None and r6 == 0 and (r12 is not None and r12 > 0):
         add(SECONDARY, "No releases in 6m but releases exist in 12m — check-in: planned pause or emerging blockers?")
 
-    if p3 is not None and p12 is not None and p12 > 0:
+    # Participation vs baseline: only compare if BOTH windows exist in the LATEST report
+    if has3 and has12 and p3 is not None and p12 is not None and p12 > 0:
         ratio = p3 / p12
         if p12 >= 15 and p3 <= 6 and ratio <= 0.25:
             add(
@@ -361,7 +415,7 @@ def build_signals_combined(row: pd.Series) -> list[dict]:
                 "Recent dev@ participation (3m) is materially lower than the 12m baseline — confirm whether discussion moved channels or participation narrowed."
             )
 
-    if m3 is not None and m12 is not None and m12 > 0:
+    if has3 and has12 and m3 is not None and m12 is not None and m12 > 0:
         drop = (m12 - m3) / m12
         if m3 <= 20 and drop >= 0.50:
             add(
@@ -369,23 +423,27 @@ def build_signals_combined(row: pd.Series) -> list[dict]:
                 "Recent dev@ message volume (3m) is lower than the 12m baseline — interpret alongside other indicators (releases/review activity)."
             )
 
-    if bus3 is not None and bus3 <= 2 and committers12 is not None and committers12 <= 10:
+    # Bus factor: needs 3m and 12m in the LATEST report
+    if has3 and has12 and bus3 is not None and bus3 <= 2 and committers12 is not None and committers12 <= 10:
         add(SECONDARY, "Contribution concentration signal (bus50 ≤ 2) with small 12m committer base — check dominance risk.")
 
-    if prs3 is not None and prs3 >= 50:
+    # PR throughput concentration: only needs 3m
+    if has3 and prs3 is not None and prs3 >= 50:
         if revdiv3 is not None and revdiv3 <= 3.0:
             add(SECONDARY, "High PR throughput but low reviewer diversity (3m) — are reviews concentrated?")
         if authdiv3 is not None and authdiv3 <= 3.0:
             add(SECONDARY, "Activity concentrated among few PR authors (3m) — is funnel broadening?")
 
-    if merge3 is not None and merge12 is not None and merge12 > 0:
+    # Merge time baseline comparison: only if BOTH windows exist in the LATEST report
+    if has3 and has12 and merge3 is not None and merge12 is not None and merge12 > 0:
         ch = (merge3 - merge12) / merge12
         if ch >= 0.50 and merge3 >= 7:
             add(SECONDARY, "3m PR merge time much slower than 12m baseline — backlog/review capacity/process change?")
         elif ch >= 0.25 and merge3 >= 7:
             add(FYI, "3m PR merge time trending slower than 12m baseline — FYI (watch next snapshot).")
 
-    if commits3 is not None and commits3 >= 100 and (p3 is not None and p3 <= 5):
+    # Commit activity vs participation: only needs 3m
+    if has3 and commits3 is not None and commits3 >= 100 and (p3 is not None and p3 <= 5):
         add(FYI, "High commit activity with low dev@ participation (3m) — FYI: check on-list socialisation norms.")
 
     return sig
@@ -399,7 +457,7 @@ def signals_summary(sig: list[dict]) -> str:
     return " • ".join([f"[{s['level']}] {s['text']}" for s in sig])
 
 
-def draft_commentary_from_combo(pod: str, row: pd.Series) -> str:
+def draft_commentary_from_latest(pod_display: str, row: pd.Series) -> str:
     sigs = row.get("signals") or []
     order = {PRIMARY: 0, SECONDARY: 1, FYI: 2}
     sigs = sorted(sigs, key=lambda x: order.get(x["level"], 9))
@@ -409,14 +467,17 @@ def draft_commentary_from_combo(pod: str, row: pd.Series) -> str:
     fyi = [s for s in sigs if s["level"] == FYI]
 
     lines: list[str] = []
-    lines.append(f"### {pod} — health cross-check summary")
+    lines.append(f"### {pod_display} — health cross-check summary")
+    lines.append("")
+    lines.append("**Data availability:**")
+    lines.extend(window_availability_messages(row))
     lines.append("")
+    lines.append("**Indicators:**")
 
-    if not sigs:
-        lines.append("No indicators triggered in the latest snapshot.")
+    if not (primary or secondary or fyi):
+        lines.append("- —")
         return "\n".join(lines)
 
-    lines.append("**Indicators:**")
     if primary:
         lines.append("- **Primary:**")
         for s in primary:
@@ -429,6 +490,7 @@ def draft_commentary_from_combo(pod: str, row: pd.Series) -> str:
         lines.append("- **FYI:**")
         for s in fyi:
             lines.append(f"  - {s['text']}")
+
     return "\n".join(lines)
 
 
@@ -499,17 +561,16 @@ def charts_panel(pod_df: pd.DataFrame):
 st.set_page_config(layout="wide")
 st.title("Incubator Health — combined view (3m/6m/12m)")
 
-if "selected_podling" not in st.session_state:
-    st.session_state["selected_podling"] = None
+if "selected_podling_key" not in st.session_state:
+    st.session_state["selected_podling_key"] = None
 
 with st.sidebar:
-    # Reports are always relative to the incubator repo root
     reports_dir = st.text_input("Reports directory", "reports")
     max_commits = st.slider("Max report runs (commits touching reports/)", 10, 300, 120, 10)
     rebuild = st.button("Rebuild dataset (clear cache)")
 
 try:
-    repo_root = resolve_incubator_repo()
+    repo_root = resolve_repo_root()
 except RuntimeError as e:
     st.error(str(e))
     st.stop()
@@ -517,16 +578,15 @@ except RuntimeError as e:
 if rebuild:
     build_dataset.clear()
 
-st.caption(f"Incubator repo: {repo_root}")
+st.caption(f"Repo: {repo_root}")
 
 df = build_dataset(str(repo_root), reports_dir, max_commits)
 
 if df.empty:
     st.warning("No data parsed. Check reports_dir.")
-    # Debug that is actually meaningful for this layout
     try:
         st.write("cwd:", str(Path.cwd().resolve()))
-        st.write("incubator repo_root:", str(repo_root))
+        st.write("repo_root:", str(repo_root))
         st.write("reports_dir:", reports_dir)
         st.write("reports_dir exists on disk:", (Path(repo_root) / reports_dir).exists())
         out = run_git(repo_root, ["ls-tree", "-r", "--name-only", "HEAD", reports_dir])
@@ -535,8 +595,9 @@ if df.empty:
         st.error(f"Debug: {e}")
     st.stop()
 
-combo = combined_snapshot(df)
-combo["signals"] = combo.apply(build_signals_combined, axis=1)
+# STRICT latest-only combo for indicators/availability
+combo = latest_combo(df)
+combo["signals"] = combo.apply(build_signals_latest_only, axis=1)
 combo["primary_count"] = combo["signals"].apply(lambda xs: sum(1 for x in xs if x["level"] == PRIMARY))
 combo["secondary_count"] = combo["signals"].apply(lambda xs: sum(1 for x in xs if x["level"] == SECONDARY))
 combo["fyi_count"] = combo["signals"].apply(lambda xs: sum(1 for x in xs if x["level"] == FYI))
@@ -545,7 +606,7 @@ combo["signal_summary"] = combo["signals"].apply(signals_summary)
 tabs = st.tabs(["📋 Podling queue", "📈 Charts", "📝 Commentary builder"])
 
 with tabs[0]:
-    st.subheader("Podling queue (combined windows)")
+    st.subheader("Podling queue (latest report per podling)")
 
     c1, c2, c3 = st.columns(3)
     with c1:
@@ -560,26 +621,28 @@ with tabs[0]:
         sort_by = st.selectbox("Sort by", ["Most primary", "Most secondary", "Podling name"], index=0)
 
     q = combo.copy()
+
     if show_level == "Primary only":
         q = q[q["primary_count"] >= 1]
     elif show_level == "Primary + Secondary":
         q = q[(q["primary_count"] + q["secondary_count"]) >= 1]
     else:
-        q = q[(q["primary_count"] + q["secondary_count"] + q["fyi_count"]) >= 1]
+        # All projects (no filtering-out by default would also be fine, but keep your UI)
+        q = q.copy()
 
     if contains.strip():
         needle = contains.strip().lower()
         q = q[q["signal_summary"].str.lower().str.contains(needle, na=False)]
 
     if sort_by == "Most primary":
-        q = q.sort_values(["primary_count", "secondary_count", "podling"], ascending=[False, False, True])
+        q = q.sort_values(["primary_count", "secondary_count", "podling_display"], ascending=[False, False, True])
     elif sort_by == "Most secondary":
-        q = q.sort_values(["secondary_count", "primary_count", "podling"], ascending=[False, False, True])
+        q = q.sort_values(["secondary_count", "primary_count", "podling_display"], ascending=[False, False, True])
     else:
-        q = q.sort_values(["podling"], ascending=[True])
+        q = q.sort_values(["podling_display"], ascending=[True])
 
     cols = [
-        "podling",
+        "podling_display",
         "primary_count",
         "secondary_count",
         "fyi_count",
@@ -604,41 +667,90 @@ with tabs[0]:
         )
         sel_rows = event.selection.rows
         if sel_rows:
-            st.session_state["selected_podling"] = q.iloc[sel_rows[0]]["podling"]
+            # map selected display back to key deterministically via the selected row in q
+            st.session_state["selected_podling_key"] = q.iloc[sel_rows[0]]["podling_key"]
     except Exception:
         st.info(
             "Row selection isn’t available in this Streamlit version. "
             "Use the dropdown below."
         )
-        st.session_state["selected_podling"] = st.selectbox(
-            "Select podling",
-            q["podling"].tolist() if len(q) else sorted(df["podling"].unique()),
-            index=0,
-            key="fallback_pod_select",
+        # fallback list: ALL projects
+        pods_all = (
+            combo[["podling_key", "podling_display"]]
+            .drop_duplicates()
+            .sort_values("podling_display")
+            .reset_index(drop=True)
         )
+        display = pods_all["podling_display"].tolist()
+        # no extra widgets elsewhere; only used when dataframe selection is unavailable
+        chosen_display = st.selectbox("Select podling", display, key="fallback_pod_select")
+        chosen_key = pods_all[pods_all["podling_display"] == chosen_display].iloc[0]["podling_key"]
+        st.session_state["selected_podling_key"] = chosen_key
 
 with tabs[1]:
     st.subheader("Charts")
 
-    pods_all = sorted(df["podling"].unique())
-
-    sel = st.session_state.get("selected_podling")
-    if sel and st.session_state.get("charts_pod") != sel:
-        st.session_state["charts_pod"] = sel
+    pods_all = (
+        combo[["podling_key", "podling_display"]]
+        .drop_duplicates()
+        .sort_values("podling_display")
+        .reset_index(drop=True)
+    )
+    options = pods_all["podling_key"].tolist()
+    fmt = dict(zip(pods_all["podling_key"], pods_all["podling_display"]))
+
+    sel_key = st.session_state.get("selected_podling_key")
+    if sel_key is None and options:
+        sel_key = options[0]
+        st.session_state["selected_podling_key"] = sel_key
+
+    # keep the charts selectbox in sync WITHOUT warnings: set session_state value, do not pass index
+    if "charts_pod_key" not in st.session_state:
+        st.session_state["charts_pod_key"] = sel_key
+    elif sel_key is not None and st.session_state["charts_pod_key"] != sel_key and sel_key in options:
+        st.session_state["charts_pod_key"] = sel_key
+
+    chosen_key = st.selectbox(
+        "Podling",
+        options,
+        format_func=lambda k: fmt.get(k, k),
+        key="charts_pod_key",
+    )
+    st.session_state["selected_podling_key"] = chosen_key
 
-    pod = st.selectbox("Podling", pods_all, key="charts_pod")
-    pod_df = df[df["podling"] == pod].sort_values("snapshot_date")
+    pod_df = df[df["podling_key"] == chosen_key].sort_values("snapshot_date")
     charts_panel(pod_df)
 
 with tabs[2]:
     st.subheader("Commentary builder (copyable)")
 
-    pods_all = sorted(df["podling"].unique())
-
-    sel = st.session_state.get("selected_podling")
-    if sel and st.session_state.get("commentary_pod") != sel:
-        st.session_state["commentary_pod"] = sel
+    pods_all = (
+        combo[["podling_key", "podling_display"]]
+        .drop_duplicates()
+        .sort_values("podling_display")
+        .reset_index(drop=True)
+    )
+    options = pods_all["podling_key"].tolist()
+    fmt = dict(zip(pods_all["podling_key"], pods_all["podling_display"]))
+
+    sel_key = st.session_state.get("selected_podling_key")
+    if sel_key is None and options:
+        sel_key = options[0]
+        st.session_state["selected_podling_key"] = sel_key
+
+    if "commentary_pod_key" not in st.session_state:
+        st.session_state["commentary_pod_key"] = sel_key
+    elif sel_key is not None and st.session_state["commentary_pod_key"] != sel_key and sel_key in options:
+        st.session_state["commentary_pod_key"] = sel_key
+
+    chosen_key = st.selectbox(
+        "Podling",
+        options,
+        format_func=lambda k: fmt.get(k, k),
+        key="commentary_pod_key",
+    )
+    st.session_state["selected_podling_key"] = chosen_key
 
-    pod = st.selectbox("Podling", pods_all, key="commentary_pod")
-    row = combo[combo["podling"] == pod].iloc[0]
-    st.code(draft_commentary_from_combo(pod, row), language="markdown")
\ No newline at end of file
+    row = combo[combo["podling_key"] == chosen_key].iloc[0]
+    pod_display = fmt.get(chosen_key, chosen_key)
+    st.code(draft_commentary_from_latest(pod_display, row), language="markdown")
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org