svn commit: r1939246 - comdev/reporter.apache.org/trunk/scripts

[se...@apache.org]

Author: sebb
Date: Sun Oct 18 13:09:05 2026
New Revision: 1939246

Log:
Report retries

Modified:
   comdev/reporter.apache.org/trunk/scripts/mailglomper2.py

Modified: comdev/reporter.apache.org/trunk/scripts/mailglomper2.py
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/mailglomper2.py	Sun Oct 18 12:53:28 2026	(r1939245)
+++ comdev/reporter.apache.org/trunk/scripts/mailglomper2.py	Sun Oct 18 13:09:05 2026	(r1939246)
@@ -122,11 +122,13 @@ async def weekly_stats(client, listpart,
     project = domainpart.replace(".apache.org", "")
     if not project:
         return {}
+    retries = 0
     try:
         mldata = await get_stats(client, listpart, domainpart, date)
     except (TimeoutError, RuntimeError) as e:
         time.sleep(20)
         tsprint("Retrying ...")
+        retries += 1
         mldata = await get_stats(client, listpart, domainpart, date)
     weekly = {}
     count = 0
@@ -139,7 +141,7 @@ async def weekly_stats(client, listpart,
                 weekly[rounded] = weekly.get(rounded, 0) + 1
             except Exception as err:
                 tsprint(err)
-    return count, weekly
+    return count, weekly, retries
 
 
 def add_weeks(total, add):
@@ -161,6 +163,7 @@ async def gather_stats():
             mlists.extend([f"{listpart}@{domainpart}" for listpart in lists])
 
     lastCheckpoint = time.time()  # when output files were last saved
+    retries = 0
     for mailinglist in mlists:
         listpart, domainpart = mailinglist.split("@", maxsplit=1)
         project = domainpart.replace(".apache.org", "")
@@ -184,7 +187,8 @@ async def gather_stats():
                 tsprint(f"Fetching data for {fetch}")
                 results = await asyncio.gather(*[weekly_stats(client, listpart, domainpart, date) for date in fetch])
             for result in results:
-                ct, weeks = result
+                ct, weeks, r = result
+                retries += r
                 add_weeks(mls[ml]["weekly"], weeks)
                 mlct += ct
                 for week in weeks:
@@ -226,8 +230,7 @@ async def gather_stats():
             json.dump(mls, f, indent=1, sort_keys=True)
 
         tsprint(f"Dumped JSON file")
-    elapsed = time.time() - startTime
-    tsprint("Completed in %d seconds" % elapsed)
+    return retries
 
 
 if __name__ == "__main__":
@@ -236,7 +239,12 @@ if __name__ == "__main__":
         # Get the index of mailing lists
         data = requests.get("https://lists.apache.org/api/preferences.json", timeout=10).json()
         tsprint("Fetched %d lists in preferences data" % len(data.get('lists',())))
-        asyncio.run(gather_stats())
+        retries = asyncio.run(gather_stats())
+        elapsed = time.time() - startTime
+        msg = "Completed in %d seconds, retries %d" % (elapsed, retries)
+        tsprint(msg)
+        if retries > 0: # send alert that retry occurred
+            print(msg, file=sys.stderr)            
     except Exception as e:
         tsprint("Failed: %s" % repr(e))
         raise # ensure it alerts the mailing list