svn commit: r1805476 - /comdev/projects.apache.org/scripts/cronjobs/parseprojects.py

[se...@apache.org]

Author: sebb
Date: Fri Aug 18 22:08:55 2017
New Revision: 1805476

URL: http://svn.apache.org/viewvc?rev=1805476&view=rev
Log:
COMDEV-222 projects.a.o does not handle SVN problems well
Only send individual mails for 404 errors

Modified:
    comdev/projects.apache.org/scripts/cronjobs/parseprojects.py

Modified: comdev/projects.apache.org/scripts/cronjobs/parseprojects.py
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/scripts/cronjobs/parseprojects.py?rev=1805476&r1=1805475&r2=1805476&view=diff
==============================================================================
--- comdev/projects.apache.org/scripts/cronjobs/parseprojects.py (original)
+++ comdev/projects.apache.org/scripts/cronjobs/parseprojects.py Fri Aug 18 22:08:55 2017
@@ -5,6 +5,7 @@ if sys.hexversion < 0x03000000:
 from xml.dom import minidom
 import xml.etree.ElementTree as ET
 import re, urllib.request
+import urllib.error
 import json
 import os
 from os.path import join
@@ -147,6 +148,7 @@ def handleChild(el):
     return tag, retval
 
 files = []
+unreportedError = False # any errors not yet mailed?
 for s in itemlist :
     url = s.childNodes[0].data
     try:
@@ -248,10 +250,16 @@ for s in itemlist :
         else:
             printMail("WARN: project ignored since unable to extract project json filename from %s" % url)
     except Exception as err:
-        if isinstance(err, OSError): # OSError is parent of HTTPError/URLError 
-            printMail("Error when processing doap file %s:" % url, file=sys.stderr, body=err.filename)
+        if isinstance(err, OSError): # OSError is parent of HTTPError/URLError
+            # Only mail 404 errors individually
+            if isinstance(err, urllib.error.HTTPError) and err.code == 404:
+                printMail("Error when processing doap file %s:" % url, file=sys.stderr, body=err.filename)
+            else:
+                print("Error when processing doap file %s:" % url, file=sys.stderr)
+                unreportedError = True
         else:
-            printMail("Error when processing doap file %s:" % url, file=sys.stderr)
+            print("Error when processing doap file %s:" % url, file=sys.stderr)
+            unreportedError = True
         print("-"*60, file=sys.stderr)
         traceback.print_exc()
         if isinstance(err, OSError): # OSError is parent of HTTPError/URLError 
@@ -287,6 +295,8 @@ if len(failures) > 0:
             f.write("<location>%s</location>\n" % fail)
         f.write("</doapFiles>\n")
         f.close()
+        if unreportedError:
+            printMail("ERROR: one or more errors detected - see the log file")    
 else:
     if os.path.exists("parseprojects-failures.xml"):
         print("No failures detected, removing previous failure data")