You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@community.apache.org by se...@apache.org on 2015/07/07 00:12:29 UTC
svn commit: r1689513 - in /comdev/reporter.apache.org/tags/test:
data/parsepmcs.py mailglomper.py parseversions.py readjira.py scandist.py
Author: sebb
Date: Mon Jul 6 22:12:29 2015
New Revision: 1689513
URL: http://svn.apache.org/r1689513
Log:
Test
Added:
comdev/reporter.apache.org/tags/test/data/parsepmcs.py (with props)
comdev/reporter.apache.org/tags/test/mailglomper.py (with props)
comdev/reporter.apache.org/tags/test/parseversions.py (with props)
comdev/reporter.apache.org/tags/test/readjira.py (with props)
comdev/reporter.apache.org/tags/test/scandist.py (with props)
Added: comdev/reporter.apache.org/tags/test/data/parsepmcs.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/data/parsepmcs.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/data/parsepmcs.py (added)
+++ comdev/reporter.apache.org/tags/test/data/parsepmcs.py Mon Jul 6 22:12:29 2015
@@ -0,0 +1,91 @@
+import re, urllib.request
+import csv
+import json
+import os
+import datetime
+import time
+pmcs = {}
+try:
+ with open("pmcs.json", "r") as f:
+ pmcs = json.loads(f.read())
+ f.close()
+except:
+ pass
+
+projects = {}
+try:
+ with open("projects.json", "r") as f:
+ projects = json.loads(f.read())
+ f.close()
+except:
+ pass
+
+
+people = {}
+newgroups = []
+
+data = urllib.request.urlopen("http://people.apache.org/committer-index.html").read().decode('utf-8')
+x = 0
+for committer in re.findall(r"<tr>([\S\s]+?)</tr>", data, re.MULTILINE | re.UNICODE):
+ x += 1
+## print(committer)
+ m = re.search(r"<a id='(.+?)'>[\s\S]+?<td.+?>\s*(.+?)</td>[\s\S]+?>(.+)</td>", committer, re.MULTILINE | re.UNICODE)
+ if m:
+ cid = m.group(1)
+ cname = re.sub(r"<.+?>", "", m.group(2), 4)
+ cproj = m.group(3)
+ isMember = False
+ if re.search(r"<b", committer, re.MULTILINE | re.UNICODE):
+ isMember = True
+ for project in re.findall(r"#([-a-z0-9._]+)-pmc", cproj):
+ now = time.time()
+ if not project in pmcs:
+ pmcs[project] = {}
+ newgroups.append(project)
+ if project in newgroups:
+ now = 0
+ if not cid in pmcs[project]:
+ pmcs[project][cid] = [cname, now, time.time()]
+ else:
+ pmcs[project][cid] = [pmcs[project][cid][0], pmcs[project][cid][1], time.time()]
+
+ for project in re.findall(r"#([-a-z0-9._]+)(?!-pmc)", cproj):
+ now = time.time()
+ if not project in projects:
+ projects[project] = {}
+ newgroups.append(project)
+ elif project in newgroups:
+ now = 0
+ if not cid in projects[project]:
+ projects[project][cid] = [cname, now, time.time()]
+ else:
+ projects[project][cid] = [projects[project][cid][0], projects[project][cid][1], time.time()]
+
+# Delete retired members
+ret = 0
+for project in projects:
+ for cid in projects[project]:
+ if len(projects[project][cid]) < 3 or projects[project][cid][2] < (time.time() - (86400*3)):
+ projects[project][cid] = "!"
+ projects[project] = {i:projects[project][i] for i in projects[project] if projects[project][i]!="!"}
+
+for project in pmcs:
+ for cid in pmcs[project]:
+ if len(pmcs[project][cid]) < 3 or pmcs[project][cid][2] < (time.time() - (86400*3)):
+ pmcs[project][cid] = "!"
+ pmcs[project] = {i:pmcs[project][i] for i in pmcs[project] if pmcs[project][i]!="!"}
+
+
+print("Writing pmcs.json")
+with open("pmcs.json", "w") as f:
+ f.write(json.dumps(pmcs))
+ f.close()
+
+print("Writing projects.json")
+with open("projects.json", "w") as f:
+ f.write(json.dumps(projects))
+ f.close()
+
+
+print("All done! removed %u retired entries" % ret)
+
Propchange: comdev/reporter.apache.org/tags/test/data/parsepmcs.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: comdev/reporter.apache.org/tags/test/mailglomper.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/mailglomper.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/mailglomper.py (added)
+++ comdev/reporter.apache.org/tags/test/mailglomper.py Mon Jul 6 22:12:29 2015
@@ -0,0 +1,67 @@
+import re, json, os, sys, urllib, time, email.utils
+
+from datetime import datetime
+
+
+mls = {}
+try:
+ with open("data/maildata_extended.json",'r') as f:
+ mls = json.loads(f.read())
+ print("Read JSON")
+except:
+ pass
+
+currentMonth = datetime.now().month
+currentYear = datetime.now().year
+after = time.time() - (86400*92)
+wayafter = time.time() - (86400*92*2)
+months = []
+for i in range(0,7):
+ date = "%04u%02u" % (currentYear, currentMonth)
+ currentMonth -= 1
+ if currentMonth == 0:
+ currentMonth = 12
+ currentYear -= 1
+ months.append(date)
+
+ now = int(time.time())
+
+data = urllib.urlopen("http://mail-archives.us.apache.org/mod_mbox/").read()
+print("Fetched %u bytes of main data" % len(data))
+y = 0
+for mlist in re.finditer(r"<a href='([-a-z0-9]+)/'", data):
+ ml = mlist.group(1)
+ y += 1
+ mls[ml] = {}
+ mls[ml]['quarterly'] = [0, 0];
+ mls[ml]['weekly'] = {}
+ for date in months:
+
+ try:
+ mldata = urllib.urlopen("http://mail-archives.us.apache.org/mod_mbox/%s/%s.mbox" % (ml, date)).read()
+ if mldata:
+ for c in re.finditer(r"Date: (.+)", mldata):
+ try:
+ d = email.utils.parsedate(c.group(1))
+ timestamp = int(time.mktime(d))
+ rounded = timestamp - (timestamp % 604800) + 604800
+ mls[ml]['weekly'][rounded] = (mls[ml]['weekly'][rounded] if rounded in mls[ml]['weekly'] else 0) + 1
+ if timestamp >= after:
+ mls[ml]['quarterly'][0] += 1
+ elif timestamp >= wayafter:
+ mls[ml]['quarterly'][1] += 1
+ except:
+ pass
+
+ except Exception as err:
+ print(err)
+ print("%s: %u" % (ml, mls[ml]['quarterly'][0]))
+ if y == 50:
+ y = 0
+ with open("data/maildata_extended.json",'w+') as f:
+ f.write(json.dumps(mls))
+with open("data/maildata_extended.json",'w+') as f:
+ f.write(json.dumps(mls))
+print("Dumped JSON")
+
+
Propchange: comdev/reporter.apache.org/tags/test/mailglomper.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: comdev/reporter.apache.org/tags/test/parseversions.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/parseversions.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/parseversions.py (added)
+++ comdev/reporter.apache.org/tags/test/parseversions.py Mon Jul 6 22:12:29 2015
@@ -0,0 +1,41 @@
+import os, sys, json, urllib2, re, time, base64
+
+def getReleaseData(project):
+ try:
+ with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as f:
+ x = json.loads(f.read())
+ f.close()
+ return x;
+ except:
+ return {}
+
+projects = {
+ 'trafficserver': 'TS',
+ 'accumulo': 'ACCUMULO'
+}
+
+jirapass = ""
+with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f:
+ jirapass = f.read().strip()
+ f.close()
+
+for project in projects:
+ jiraname = projects[project]
+ base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1]
+ rdata = getReleaseData(project)
+ try:
+ req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project/%s/versions" % jiraname)
+ req.add_header("Authorization", "Basic %s" % base64string)
+ cdata = json.loads(urllib2.urlopen(req).read())
+ for entry in cdata:
+ if ('name' in entry and 'releaseDate' in entry and 'released' in entry and entry['released']):
+
+ date = time.mktime(time.strptime(entry['releaseDate'], "%Y-%m-%d"))
+ print("Updating version for %s - %s: %u" % (project, entry['name'], date))
+ rdata[entry['name']] = date
+ except Exception as err:
+ print(err)
+
+ with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "w") as f:
+ f.write(json.dumps(rdata))
+ f.close()
\ No newline at end of file
Propchange: comdev/reporter.apache.org/tags/test/parseversions.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: comdev/reporter.apache.org/tags/test/readjira.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/readjira.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/readjira.py (added)
+++ comdev/reporter.apache.org/tags/test/readjira.py Mon Jul 6 22:12:29 2015
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import re, os, json, urllib2, base64, time
+from os import listdir
+from os.path import isfile, join
+
+mypath = "/var/www/reporter.apache.org/data/JIRA"
+myfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ]
+
+jirapass = ""
+with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f:
+ jirapass = f.read().strip()
+ f.close()
+
+def getJIRAS(project):
+ refresh = True
+ if refresh:
+ base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1]
+
+ try:
+ req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20created%20%3E=%20-91d""")
+ req.add_header("Authorization", "Basic %s" % base64string)
+ cdata = json.loads(urllib2.urlopen(req).read())
+ req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20resolved%20%3E=%20-91d""")
+ req.add_header("Authorization", "Basic %s" % base64string)
+ rdata = json.loads(urllib2.urlopen(req).read())
+ with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f:
+ f.write(json.dumps([cdata['total'], rdata['total'], project]))
+ f.close()
+ return cdata['total'], rdata['total'], project
+ except Exception as err:
+ with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f:
+ f.write(json.dumps([0,0,None]))
+ f.close()
+ return 0,0, None
+
+for project in myfiles:
+ jiraname = project.replace(".json", "")
+ if jiraname != "projects":
+ print("Refreshing JIRA stats for " + jiraname)
+ getJIRAS(jiraname)
+ time.sleep(2)
+
\ No newline at end of file
Propchange: comdev/reporter.apache.org/tags/test/readjira.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: comdev/reporter.apache.org/tags/test/scandist.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/scandist.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/scandist.py (added)
+++ comdev/reporter.apache.org/tags/test/scandist.py Mon Jul 6 22:12:29 2015
@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+
+#################################################
+# scandist.py: a minimalistic svnwcsub program #
+#################################################
+
+from threading import Thread
+from datetime import datetime
+import os
+import sys
+
+
+# SMTP Lib
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+version = 2
+if sys.hexversion < 0x03000000:
+ print("Using Python 2...")
+ import json, httplib, urllib, urllib2, ConfigParser as configparser, re, base64, sys, os, time, atexit, signal, logging, socket, subprocess
+ socket._fileobject.default_bufsize = 0
+else:
+ print("Using Python 3")
+ version = 3
+ import json, http.client, urllib.request, urllib.parse, configparser, re, base64, sys, os, time, atexit, signal, logging, subprocess
+
+targets = {}
+###########################################################
+# Daemon class, curtesy of an anonymous good-hearted soul #
+###########################################################
+class daemon:
+ """A generic daemon class.
+
+ Usage: subclass the daemon class and override the run() method."""
+
+ def __init__(self, pidfile): self.pidfile = pidfile
+
+ def daemonize(self):
+ """Deamonize class. UNIX double fork mechanism."""
+
+ try:
+ pid = os.fork()
+ if pid > 0:
+ # exit first parent
+ sys.exit(0)
+ except OSError as err:
+ sys.stderr.write('fork #1 failed: {0}\n'.format(err))
+ sys.exit(1)
+
+ # decouple from parent environment
+ os.chdir('/')
+ os.setsid()
+ os.umask(0)
+
+ # do second fork
+ try:
+ pid = os.fork()
+ if pid > 0:
+
+ # exit from second parent
+ sys.exit(0)
+ except OSError as err:
+ sys.stderr.write('fork #2 failed: {0}\n'.format(err))
+ sys.exit(1)
+
+ # redirect standard file descriptors
+ sys.stdout.flush()
+ sys.stderr.flush()
+ si = open(os.devnull, 'r')
+ so = open(os.devnull, 'a+')
+ se = open(os.devnull, 'a+')
+
+ os.dup2(si.fileno(), sys.stdin.fileno())
+ os.dup2(so.fileno(), sys.stdout.fileno())
+ os.dup2(se.fileno(), sys.stderr.fileno())
+
+ # write pidfile
+ atexit.register(self.delpid)
+
+ pid = str(os.getpid())
+ with open(self.pidfile,'w+') as f:
+ f.write(pid + '\n')
+
+ def delpid(self):
+ os.remove(self.pidfile)
+
+ def start(self):
+ """Start the daemon."""
+
+ # Check for a pidfile to see if the daemon already runs
+ try:
+ with open(self.pidfile,'r') as pf:
+
+ pid = int(pf.read().strip())
+ except IOError:
+ pid = None
+
+ if pid:
+ message = "pidfile {0} already exist. " + \
+ "Daemon already running?\n"
+ sys.stderr.write(message.format(self.pidfile))
+ sys.exit(1)
+
+ # Start the daemon
+ self.daemonize()
+ self.run()
+
+ def stop(self):
+ """Stop the daemon."""
+
+ # Get the pid from the pidfile
+ try:
+ with open(self.pidfile,'r') as pf:
+ pid = int(pf.read().strip())
+ except IOError:
+ pid = None
+
+ if not pid:
+ message = "pidfile {0} does not exist. " + \
+ "Daemon not running?\n"
+ sys.stderr.write(message.format(self.pidfile))
+ return # not an error in a restart
+
+ # Try killing the daemon process
+ try:
+ while 1:
+ os.kill(pid, signal.SIGTERM)
+ time.sleep(0.1)
+ except OSError as err:
+ e = str(err.args)
+ if e.find("No such process") > 0:
+ if os.path.exists(self.pidfile):
+ os.remove(self.pidfile)
+ else:
+ print (str(err.args))
+ sys.exit(1)
+
+ def restart(self):
+ """Restart the daemon."""
+ self.stop()
+ self.start()
+
+ def run(self):
+ """You should override this method when you subclass Daemon.
+
+ It will be called after the process has been daemonized by
+ start() or restart()."""
+
+
+
+####################
+# Helper functions #
+####################
+
+
+# read_chunk: iterator for reading chunks from the stream
+# since this is all handled via urllib now, this is quite rudimentary
+def read_chunk(req):
+ while True:
+ try:
+ line = req.readline().strip()
+ if line:
+ yield line
+ else:
+ print("No more lines?")
+ break
+ except Exception as info:
+
+ break
+ return
+
+
+#########################
+# Main listener program #
+#########################
+
+
+
+# PubSub class: handles connecting to a pubsub service and checking commits
+class PubSubClient(Thread):
+ def run(self):
+ global targets
+ while True:
+
+ self.req = None
+ while not self.req:
+ try:
+ if version == 3:
+ self.req = urllib.request.urlopen(self.url, None, 30)
+ else:
+ self.req = urllib2.urlopen(self.url, None, 30)
+
+ except:
+
+ time.sleep(30)
+ continue
+
+ for line in read_chunk(self.req):
+ if version == 3:
+ line = str( line, encoding='ascii' ).rstrip('\r\n,').replace('\x00','') # strip away any old pre-0.9 commas from gitpubsub chunks and \0 in svnpubsub chunks
+ else:
+ line = str( line ).rstrip('\r\n,').replace('\x00','') # strip away any old pre-0.9 commas from gitpubsub chunks and \0 in svnpubsub chunks
+ try:
+ obj = json.loads(line)
+ if "commit" in obj and "repository" in obj['commit']:
+ obj['commit']['whence'] = time.time()
+
+
+ # If it's our public svn repo, then...
+ if 'changed' in obj['commit']:
+
+ #Grab some vars
+ commit = obj['commit']
+ body = commit['log']
+ svnuser = commit['committer']
+ path, action = commit['changed'].popitem()
+ revision = commit['id']
+ email = svnuser + "@apache.org"
+
+ # Figure out the root thingermajig
+ match = re.match(r"^release/([a-z0-9]+)", path)
+ if match:
+ if match.group(1) != "incubator":
+ targets[svnuser] = match.group(1)
+
+
+
+ except ValueError as detail:
+ continue
+
+
+
+
+
+################
+# Main program #
+################
+def main():
+ global targets
+ if debug:
+ print("Foreground test mode enabled, no updates will be made")
+
+
+
+ # Start the svn thread
+ svn_thread = PubSubClient()
+ svn_thread.url = "http://hades.apache.org:2069/commits/*"
+ svn_thread.start()
+
+ while True:
+ now = time.time()
+ then = now - 86400
+
+ time.sleep(600)
+ targetstwo = targets
+ targets = {}
+ for user in targetstwo:
+ email = user + "@apache.org"
+ project = targetstwo[user]
+ sender = 'no-reply@reporter.apache.org'
+ receivers = [email, 'humbedooh@apache.org']
+ print("Notifying %s of new data pushed to %s" % (email, project))
+ message = """From: Apache Reporter Service <no...@reporter.apache.org>
+To: %s <%s>
+Reply-To: dev@community.apache.org
+Subject: Please add your release data for '%s'
+
+Hi,
+This is an automated email from reporter.apache.org.
+I see that you just pushed something to our release repository for the %s project.
+
+If you are a PMC member of this project, we ask that you log on to:
+https://reporter.apache.org/addrelease.html?%s
+and add your release data (version and date) to the database.
+
+If you are not a PMC member, please have a PMC member add this information.
+
+While this is not a requirement, we ask that you still add this data to the
+reporter database, so that people using the Apache Reporter Service will be
+able to see the latest release data for this project.
+
+With regards,
+The Apache Reporter Service.
+ """ % (user, email, project, project, project)
+
+ try:
+ smtpObj = smtplib.SMTP('localhost')
+ smtpObj.sendmail(sender, receivers, message)
+ print "Successfully sent email"
+ except SMTPException:
+ print "Error: unable to send email"
+
+
+##############
+# Daemonizer #
+##############
+class MyDaemon(daemon):
+ def run(self):
+ main()
+
+if __name__ == "__main__":
+ daemon = MyDaemon('/tmp/scandist.pid')
+ if len(sys.argv) >= 2:
+ if 'start' == sys.argv[1]:
+ daemon.start()
+ elif 'stop' == sys.argv[1]:
+ daemon.stop()
+ elif 'restart' == sys.argv[1]:
+ daemon.restart()
+ elif 'foreground' == sys.argv[1]:
+ debug = True
+ main()
+ else:
+ print("Unknown command")
+ sys.exit(2)
+ sys.exit(0)
+ else:
+ print("usage: %s start|stop|restart|foreground" % sys.argv[0])
+ sys.exit(2)
+
Propchange: comdev/reporter.apache.org/tags/test/scandist.py
------------------------------------------------------------------------------
svn:eol-style = native