You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@community.apache.org by hu...@apache.org on 2019/08/08 11:07:48 UTC
svn commit: r1864689 - in /comdev/reporter.apache.org/trunk/scripts:
pdata.py rapp/ rapp/drafts.py rapp/overview.py rapp/whimsy.py wsgi.py
Author: humbedooh
Date: Thu Aug 8 11:07:48 2019
New Revision: 1864689
URL: http://svn.apache.org/viewvc?rev=1864689&view=rev
Log:
Add in new API endpoints for WSGI app.
Added:
comdev/reporter.apache.org/trunk/scripts/rapp/
comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py
comdev/reporter.apache.org/trunk/scripts/rapp/overview.py
comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py
Modified:
comdev/reporter.apache.org/trunk/scripts/pdata.py
comdev/reporter.apache.org/trunk/scripts/wsgi.py
Modified: comdev/reporter.apache.org/trunk/scripts/pdata.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/pdata.py?rev=1864689&r1=1864688&r2=1864689&view=diff
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/pdata.py (original)
+++ comdev/reporter.apache.org/trunk/scripts/pdata.py Thu Aug 8 11:07:48 2019
@@ -41,6 +41,9 @@ MEMBER_INFO = 'https://whimsy.apache.org
PROJECTS = 'https://whimsy.apache.org/public/public_ldap_projects.json'
DESCRIPTIONS = 'https://projects.apache.org/json/foundation/committees.json'
+def has_cache(filename, ttl = 14400):
+ return (os.path.exists(filename) and os.path.getmtime(filename) > (time.time() - ttl))
+
jmap = {
'trafficserver': ['TS'],
'cordova': ['CB'],
Added: comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py?rev=1864689&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py Thu Aug 8 11:07:48 2019
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+""" script for working with drafts """
+import os
+import sys
+import time
+import json
+import re
+import pdata
+import committee_info
+
+DRAFTS_DIR = '/tmp/rapp-drafts'
+EDITOR_TYPE = 'unified'
+
+if not os.path.isdir(DRAFTS_DIR):
+ os.makedirs(DRAFTS_DIR, exist_ok = True)
+
+def has_access(user, project):
+ member = pdata.isASFMember(user)
+ pmc = project in pdata.getPMCs(user)
+ return (member or pmc)
+
+def index(environ, user):
+ """ Listy currently existing drafts for a project """
+ project = environ.get('QUERY_STRING')
+ drafts = {}
+
+ if has_access(user, project):
+ whence = int(time.time() - (60*86400)) # Max 2 months ago!
+ for filename in [x for x in os.listdir(DRAFTS_DIR) if x.startswith(EDITOR_TYPE) and x.endswith('.draft')]:
+ e, p, t, u = filename.split('-', 3)
+ t = int(t)
+ # If a file is way old, try deleting it.
+ if t < whence:
+ try:
+ os.unlink("%s/%s" % (DRAFTS_DIR, filename))
+ except:
+ pass
+ elif p == project and t >= whence:
+ u = u.replace('.draft', '')
+ drafts[t] = {'filename': filename, 'creator': u, 'yours': user == u}
+
+ return { 'drafts': drafts }
+
+def fetch(environ, user):
+ """ Fetch a draft if access is right... """
+ filename = environ.get('QUERY_STRING')
+ m = re.match(r"[^-./]+-([^-./]+)-\d+-[^-./]+\.draft$", filename)
+ if not m:
+ return {'error': "Invalid filename!"}
+ if os.path.exists(os.path.join(DRAFTS_DIR, filename)):
+ project = m.group(1)
+ if has_access(user, project):
+ report = open(os.path.join(DRAFTS_DIR, filename), "r").read()
+ return {'report': report}
+ return {}
+
+def delete(environ, user):
+ """ Delete a draft if access is right... """
+ filename = environ.get('QUERY_STRING')
+ m = re.match(r"[^-./]+-[^-./]+-\d+-([^-/]+)\.draft$", filename)
+ if not m:
+ return {'error': "Invalid filename!"}
+ if os.path.exists(os.path.join(DRAFTS_DIR, filename)):
+ u = m.group(1)
+ if user == u:
+ try:
+ os.unlink(os.path.join(DRAFTS_DIR, filename))
+ return {'message': 'Draft deleted'}
+ except:
+ pass
+ return {'error': "Could not delete draft!"}
+
+def save(environ, user):
+ """ Save a draft """
+ try:
+ request_body_size = int(environ.get('CONTENT_LENGTH', 0))
+ except (ValueError):
+ request_body_size = 0
+ if request_body_size:
+ request_body = environ['wsgi.input'].read(request_body_size)
+ try:
+ js = json.loads(request_body.decode('utf-8'))
+ except:
+ js = {}
+ if js:
+ project = js.get('project')
+ if has_access(user, project):
+ report = js.get('report')
+ now = int(time.time())
+ filename = '%s-%s-%s-%s.draft' % (EDITOR_TYPE, project, now, user)
+ try:
+ with open(os.path.join(DRAFTS_DIR, filename), "w") as f:
+ f.write(report)
+ f.close()
+ return {
+ 'okay': True,
+ 'filename': filename,
+ }
+ except:
+ return {
+ 'okay': False,
+ 'error': 'Could not save draft (permission issue?? disk full?)',
+ }
+ else:
+ return {
+ 'okay': False,
+ 'error': 'You do not have access to this project',
+ }
+ else:
+ return {
+ 'okay': False,
+ 'error': "Invalid data!",
+ }
Added: comdev/reporter.apache.org/trunk/scripts/rapp/overview.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/overview.py?rev=1864689&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/overview.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/overview.py Thu Aug 8 11:07:48 2019
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import os
+import cgi
+import json
+import pdata
+import time
+import re
+import committee_info
+
+CACHE_TIMEOUT = 14400
+
+def run(environ, user):
+ committers = pdata.loadJson(pdata.COMMITTER_INFO)['people']
+ pmcSummary = committee_info.PMCsummary()
+ project = environ.get('QUERY_STRING')
+ output = {'okay': False, 'error': 'Unknown user ID provided!'}
+
+ dumps = {}
+ groups = []
+ if user:
+ groups = pdata.getPMCs(user)
+ if project and user and re.match(r"[-a-z0-9]+", project):
+ groups = [project]
+
+ for xproject in groups:
+
+ # Try cache first? (max 6 hours old)
+ wanted_file = "/tmp/pdata-%s.json" % xproject
+ if xproject == project:
+ wanted_file = "/tmp/pdata-kibbled-%s.json" % xproject
+ if (os.path.exists(wanted_file) and os.path.getmtime(wanted_file) > (time.time() - CACHE_TIMEOUT)):
+ mpdata = json.load(open(wanted_file, "r"))
+ # If cache failed, generate fom scratch
+ else:
+ mpdata = pdata.generate(user, xproject, xproject == project)
+ if not mpdata:
+ break
+ open(wanted_file, "w").write(json.dumps(mpdata))
+ # Weave results into combined object, mindful of kibble data
+ for k, v in mpdata.items():
+ if k not in dumps:
+ dumps[k] = {}
+ if (k != 'kibble'):
+ dumps[k][xproject] = v
+ if k == 'kibble' and v:
+ dumps['kibble'] =v
+
+ # Set personalized vars, dump
+ if dumps and user:
+ ddata, allpmcs, health = pdata.getProjectData()
+ dumps['you'] = committers[user]
+ dumps['all'] = sorted(allpmcs)
+ dumps['pmcs'] = sorted(groups)
+ dumps['pmcsummary'] = pmcSummary
+ output = dumps
+
+ return output
Added: comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py?rev=1864689&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py Thu Aug 8 11:07:48 2019
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+""" script for publishing a report to whimsy """
+import os
+import sys
+import time
+import json
+import requests
+import re
+import pdata
+import committee_info
+
+WHIMSY_SUBMIT = 'https://whimsy.apache.org/board/agenda/json/post'
+WHIMSY_AGENDA = 'https://whimsy.apache.org/board/agenda/latest.json'
+WHIMSY_COMMENTS = 'https://whimsy.apache.org/board/agenda/json/historical-comments'
+
+def get_whimsy(url, env, ttl = 14400):
+ cached = True
+ xurl = re.sub(r"[^a-z0-9]+", "-", url.replace('.json', ''))
+ wanted_file = '/tmp/%s.json' % xurl
+ if pdata.has_cache(wanted_file, ttl = ttl):
+ js = json.load(open(wanted_file))
+ else:
+ try:
+ print("Fetching %s => %s..." % (url, wanted_file))
+ js = requests.get(url, headers = {'Authorization': env.get('HTTP_AUTHORIZATION')}, timeout = 5).json()
+ with open(wanted_file, "w") as f:
+ json.dump(js, f)
+ f.close()
+ cached = False
+ except: # fall back to cache on failure!
+ js = json.load(open(wanted_file))
+
+ return js, cached
+
+def has_access(user, project):
+ member = pdata.isASFMember(user)
+ pmc = project in pdata.getPMCs(user)
+ return (member or pmc)
+
+def guess_title(project):
+ """ Guess the whimsy name of a project """
+ pmcSummary = committee_info.PMCsummary()
+
+ # Figure out the name as written in whimsy..
+ pname = project
+ if project in pmcSummary:
+ pname = pmcSummary[project]['name'].replace('Apache ', '')
+
+ return pname
+
+def agenda_forced(environ, user):
+ """ Force whimsy agenda refresh... """
+ get_whimsy(WHIMSY_AGENDA, environ, ttl = 0)
+ return agenda(environ, user)
+
+def agenda(environ, user):
+ """ Returns data on the board report for a project, IF present and/or filed in the current agenda """
+ project = environ.get('QUERY_STRING')
+ report = None
+ if has_access(user, project):
+ agenda, cached = get_whimsy(WHIMSY_AGENDA, environ, ttl = 3600)
+ for entry in agenda:
+ ml = entry.get('mail_list') # mailing list id, usually correct
+ rid = entry.get('roster', '').replace('https://whimsy.apache.org/roster/committee/', '') # ldap id per roster
+ if ml and (ml == project or rid == project):
+ report = entry
+ break
+
+ comments, cached = get_whimsy(WHIMSY_COMMENTS, environ)
+ title = report and report.get('title') or guess_title(project)
+ if title in comments:
+ comments = comments[title]
+ else:
+ comments = {}
+
+ return {
+ 'can_access': True,
+ 'found': report and True or False,
+ 'filed': report and report.get('report') and True or False,
+ 'report': report,
+ 'comments': comments,
+ }
+
+ return {
+ 'can_access': False,
+ 'found': False,
+ }
+
+
+ comments, cached = get_whimsy(WHIMSY_COMMENTS, environ)
+
+def comments(environ, user):
+ """ Display board feedback from previous reports ... """
+ project = environ.get('QUERY_STRING')
+ comments, cached = get_whimsy(WHIMSY_COMMENTS, environ)
+
+ pmcSummary = committee_info.PMCsummary()
+
+ # Figure out the name as written in whimsy..
+ pname = project
+ if project in pmcSummary:
+ pname = pmcSummary[project]['name'].replace('Apache ', '')
+ cmt = {}
+
+ # If we can access, fetch comments
+ if comments and pname in comments and has_access(user, project):
+ comments = comments[pname]
+ else:
+ comments = {}
+
+ js = {
+ "pid": project,
+ "pname": pname,
+ "comments": comments,
+ "used_cache": cached,
+ }
+ return js
+
+
+def publish(environ, user):
+ try:
+ request_body_size = int(environ.get('CONTENT_LENGTH', 0))
+ except (ValueError):
+ request_body_size = 0
+ if request_body_size:
+ request_body = environ['wsgi.input'].read(request_body_size)
+ try:
+ js = json.loads(request_body.decode('utf-8'))
+ except:
+ js = {}
+ if js:
+ agenda = js.get('agenda')
+ project = js.get('project')
+ report = js.get('report')
+ digest = js.get('digest')
+ attach = js.get('attach')
+ print(project, agenda)
+ if agenda and project and report:
+ message = "Publishing report for %s via Reporter" % project
+ payload = {
+ 'agenda': agenda,
+ 'project': project,
+ 'report': report,
+ 'message': message,
+ }
+ if digest and attach:
+ del payload['project']
+ payload['attach'] = attach
+ payload['message'] = "Updating report for %s via Reporter." % project
+ payload['digest'] = digest
+ try:
+ rv = requests.post(WHIMSY_SUBMIT, headers = {
+ 'Authorization': environ.get('HTTP_AUTHORIZATION'),
+ "Content-Type": "application/json"
+ }, json = payload, timeout = 10)
+ rv.raise_for_status()
+ print(rv.text)
+ return {'okay': True, 'message': "Posted to board agenda!"}
+ except:
+ pass
+ return {}
+
+
\ No newline at end of file
Modified: comdev/reporter.apache.org/trunk/scripts/wsgi.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/wsgi.py?rev=1864689&r1=1864688&r2=1864689&view=diff
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/wsgi.py (original)
+++ comdev/reporter.apache.org/trunk/scripts/wsgi.py Thu Aug 8 11:07:48 2019
@@ -1,62 +1,42 @@
#!/usr/bin/env python2.7
-import os
-import cgi
import json
-import pdata
import time
+import base64
import re
-import committee_info
CACHE_TIMEOUT = 14400
+import rapp.overview
+import rapp.whimsy
+import rapp.drafts
+
+webmap = {
+ '/api/overview': rapp.overview.run,
+ '/api/whimsy/comments': rapp.whimsy.comments,
+ '/api/whimsy/agenda': rapp.whimsy.agenda,
+ '/api/whimsy/agenda/refresh': rapp.whimsy.agenda_forced,
+ '/api/whimsy/publish': rapp.whimsy.publish,
+ '/api/drafts/index': rapp.drafts.index,
+ '/api/drafts/save': rapp.drafts.save,
+ '/api/drafts/fetch': rapp.drafts.fetch,
+ '/api/drafts/delete': rapp.drafts.delete,
+}
def app(environ, start_fn):
- committers = pdata.loadJson(pdata.COMMITTER_INFO)['people']
- pmcSummary = committee_info.PMCsummary()
- project = environ.get('QUERY_STRING')
- user = environ.get('HTTP_X_AUTHENTICATED_USER', 'humbedooh')
-
- output = {'okay': False, 'error': 'Unknown user ID provided!'}
-
- dumps = {}
- groups = []
- if user:
- groups = pdata.getPMCs(user)
- if project and user and re.match(r"[-a-z0-9]+", project):
- groups = [project]
-
- for xproject in groups:
-
- # Try cache first? (max 6 hours old)
- wanted_file = "/tmp/pdata-%s.json" % xproject
- if xproject == project:
- wanted_file = "/tmp/pdata-kibbled-%s.json" % xproject
- if (os.path.exists(wanted_file) and os.path.getmtime(wanted_file) > (time.time() - CACHE_TIMEOUT)):
- mpdata = json.load(open(wanted_file, "r"))
- # If cache failed, generate fom scratch
- else:
- mpdata = pdata.generate(user, xproject, xproject == project)
- if not mpdata:
- break
- open(wanted_file, "w").write(json.dumps(mpdata))
- # Weave results into combined object, mindful of kibble data
- for k, v in mpdata.items():
- if k not in dumps:
- dumps[k] = {}
- if (k != 'kibble'):
- dumps[k][xproject] = v
- if k == 'kibble' and v:
- dumps['kibble'] =v
-
- # Set personalized vars, dump
- if dumps and user:
- ddata, allpmcs, health = pdata.getProjectData()
- dumps['you'] = committers[user]
- dumps['all'] = sorted(allpmcs)
- dumps['pmcs'] = sorted(groups)
- dumps['pmcsummary'] = pmcSummary
- output = dumps
+ now = time.time()
+ bauth = re.match(r"Basic (.+)", environ.get('HTTP_AUTHORIZATION', 'foo'))
+ if bauth:
+ bdec = base64.b64decode(bauth.group(1)).decode('utf-8')
+ m = re.match("^(.+?):(.+)$", bdec)
+ if m:
+ user = m.group(1)
+ uri = environ.get('PATH_INFO', '/')
+ if uri in webmap:
+ output = webmap[uri](environ, user)
+ else:
+ output = {'okay': False, 'message': 'Unknown URI %s' % uri}
+ output['took'] = int((time.time() - now) * 1000)
out = json.dumps(output, indent = 2, sort_keys = True).encode('ascii')
start_fn('200 OK', [('Content-Type', 'application/json'), ('Content-Length', str(len(out)))])
return [out]