You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@community.apache.org by hb...@apache.org on 2015/05/09 18:51:29 UTC
svn commit: r1678501 [1/2] - in /comdev/projects.apache.org: scripts/
scripts/cronjobs/ site/js/ site/json/foundation/
Author: hboutemy
Date: Sat May 9 16:51:29 2015
New Revision: 1678501
URL: http://svn.apache.org/r1678501
Log:
renamed committers.json to groups.json and projects field to groups to better match data source, ie LDAP groups
Added:
comdev/projects.apache.org/site/json/foundation/groups.json
- copied unchanged from r1678500, comdev/projects.apache.org/site/json/foundation/committers.json
Removed:
comdev/projects.apache.org/site/json/foundation/committers.json
Modified:
comdev/projects.apache.org/scripts/README.txt
comdev/projects.apache.org/scripts/cronjobs/parsecommitters.py
comdev/projects.apache.org/site/js/projects.js
comdev/projects.apache.org/site/json/foundation/.htaccess
comdev/projects.apache.org/site/json/foundation/HEADER.html
comdev/projects.apache.org/site/json/foundation/people.json
Modified: comdev/projects.apache.org/scripts/README.txt
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/scripts/README.txt?rev=1678501&r1=1678500&r2=1678501&view=diff
==============================================================================
--- comdev/projects.apache.org/scripts/README.txt (original)
+++ comdev/projects.apache.org/scripts/README.txt Sat May 9 16:51:29 2015
@@ -14,8 +14,8 @@ various sources:
- parsecommitters.py: Fetches and parses the committer (LDAP) list via
people.apache.org.
in: http://people.apache.org/committer-index.html
- out: foundation/people.json + foundation/committers.json
- List of committers with reference to projects (people.json) and pmcs with corresponding committers (committers.json)
+ out: foundation/people.json + foundation/groups.json
+ List of committers with reference to groups (people.json) and groups with corresponding committers (groups.json)
- podlings.py: Reads podlings.xml from the incubator site and creates a JSON
with timeline data, as well as current podling projects information.
@@ -50,4 +50,4 @@ various sources:
list of PMCs with site url (pmcs.json) and monthly list of new committees (committees.json)
-Notice: cycles.json is not imported from anywhere
+Notice: cycles.json is not imported from anywhere (could/should be from committee-info.txt)
Modified: comdev/projects.apache.org/scripts/cronjobs/parsecommitters.py
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/scripts/cronjobs/parsecommitters.py?rev=1678501&r1=1678500&r2=1678501&view=diff
==============================================================================
--- comdev/projects.apache.org/scripts/cronjobs/parsecommitters.py (original)
+++ comdev/projects.apache.org/scripts/cronjobs/parsecommitters.py Sat May 9 16:51:29 2015
@@ -1,12 +1,9 @@
-from xml.dom import minidom
-import xml.etree.ElementTree as ET
-import re, urllib.request
-import csv
+import re
+import urllib.request
import json
-import os
people = {}
-pmcs = {}
+groups = {}
data = urllib.request.urlopen("http://people.apache.org/committer-index.html").read().decode('utf-8')
x = 0
for committer in re.findall(r"<tr>([\S\s]+?)</tr>", data, re.MULTILINE | re.UNICODE):
@@ -16,19 +13,19 @@ for committer in re.findall(r"<tr>([\S\s
if m:
cid = m.group(1)
cname = re.sub(r"<.+?>", "", m.group(2), 4)
- cproj = m.group(3)
- projects = []
+ groupsString = m.group(3)
+ cgroups = []
isMember = False
if re.search(r"<b", committer, re.MULTILINE | re.UNICODE):
isMember = True
- for project in re.findall(r"#([-a-z0-9._]+)", cproj):
- projects.append(project)
- pmcs[project] = pmcs[project] if project in pmcs else []
- pmcs[project].append(cid)
+ for group in re.findall(r"#([-a-z0-9._]+)", groupsString):
+ cgroups.append(group)
+ groups[group] = groups[group] if group in groups else []
+ groups[group].append(cid)
people[cid] = {
'name': cname,
'member': isMember,
- 'projects': projects
+ 'groups': cgroups
}
@@ -37,9 +34,9 @@ with open("../../site/json/foundation/pe
f.write(json.dumps(people, sort_keys=True, indent=0))
f.close()
-print("Writing committers.json")
-with open("../../site/json/foundation/committers.json", "w") as f:
- f.write(json.dumps(pmcs, sort_keys=True, indent=0))
+print("Writing groups.json")
+with open("../../site/json/foundation/groups.json", "w") as f:
+ f.write(json.dumps(groups, sort_keys=True, indent=0))
f.close()
print("All done!")
\ No newline at end of file
Modified: comdev/projects.apache.org/site/js/projects.js
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/site/js/projects.js?rev=1678501&r1=1678500&r2=1678501&view=diff
==============================================================================
--- comdev/projects.apache.org/site/js/projects.js (original)
+++ comdev/projects.apache.org/site/js/projects.js Sat May 9 16:51:29 2015
@@ -1238,7 +1238,7 @@ function weaveInProjects(json) {
}
// Add committer IDs+names to project hashes
-function weaveInCommitters(json) {
+function weaveInGroups(json) {
unixgroups = json
for (p in json) {
if (projects[p] && p != 'incubator') {
@@ -1327,7 +1327,7 @@ function preloadEverything(callback) {
["/json/foundation/committees.json", setCommittees],
["/json/foundation/projects.json", weaveInProjects],
["/json/foundation/podlings.json", weaveInProjects],
- ["/json/foundation/committers.json", weaveInCommitters],
+ ["/json/foundation/groups.json", weaveInGroups],
["/json/foundation/releases.json", function(json) { releases = json; }],
],
callback);
Modified: comdev/projects.apache.org/site/json/foundation/.htaccess
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/site/json/foundation/.htaccess?rev=1678501&r1=1678500&r2=1678501&view=diff
==============================================================================
--- comdev/projects.apache.org/site/json/foundation/.htaccess (original)
+++ comdev/projects.apache.org/site/json/foundation/.htaccess Sat May 9 16:51:29 2015
@@ -2,9 +2,9 @@ IndexOptions FancyIndexing DescriptionWi
AddDescription "generated by <code>cronjobs/countaccounts.py</code>" accounts.json
AddDescription "generated by <code>cronjobs/parsechairs.py</code>" chairs.json
AddDescription "imported with <code>import/parsecommittees.py</code>" committees.json
-AddDescription "generated by <code>cronjobs/parsecommitters.py</code>" committers.json
AddDescription "manually maintained" cycles.json
AddDescription "generated by <code>cronjobs/podlings.py</code>" evolution.json
+AddDescription "generated by <code>cronjobs/parsecommitters.py</code>" groups.json
AddDescription "generated by <code>cronjobs/parsecommitters.py</code>" people.json
AddDescription "imported with <code>import/parsepmcs.py</code> from <a href='https://svn.apache.org/repos/asf/infrastructure/site-tools/trunk/projects/data_files/'>PMC DOAP files</a>" pmcs.json
AddDescription "generated by <code>cronjobs/podlings.py</code>" podlings.json
Modified: comdev/projects.apache.org/site/json/foundation/HEADER.html
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/site/json/foundation/HEADER.html?rev=1678501&r1=1678500&r2=1678501&view=diff
==============================================================================
--- comdev/projects.apache.org/site/json/foundation/HEADER.html (original)
+++ comdev/projects.apache.org/site/json/foundation/HEADER.html Sat May 9 16:51:29 2015
@@ -6,9 +6,9 @@ for more information.
<li><a href="accounts.json"><code>accounts.json</code></a> generated by <code>cronjobs/countaccounts.py</code> from LDAP</li>
<li><a href="chairs.json"><code>chairs.json</code></a> generated by <code>cronjobs/parsechairs.py</code> from <a href="http://www.apache.org/foundation/">http://www.apache.org/foundation/</a></li>
<li><a href="committees.json"><code>committees.json</code></a> imported with <code>import/parsecommittees.py</code> from <code>https://svn.apache.org/repos/private/committers/board/committee-info.txt</code></li>
-<li><a href="committers.json"><code>committers.json</code></a> and <a href="people.json"><code>people.json</code></a> generated by <code>cronjobs/parsecommitters.py</code> from <a href="http://people.apache.org/committer-index.html">http://people.apache.org/committer-index.html</a></li>
<li><a href="cycles.json"><code>cycles.json</code></a> manually maintained</li>
<li><a href="evolution.json"><code>evolution.json</code></a> and <a href="podlings.json"><code>podlings.json</code></a> generated by <code>cronjobs/podlings.py</code> from <a href="http://incubator.apache.org/podlings.xml">http://incubator.apache.org/podlings.xml</a></li>
+<li><a href="groups.json"><code>groups.json</code></a> and <a href="people.json"><code>people.json</code></a> generated by <code>cronjobs/parsecommitters.py</code> from <a href="http://people.apache.org/committer-index.html">http://people.apache.org/committer-index.html</a></li>
<li><a href="pmcs.json"><code>pmcs.json</code></a> imported with <code>import/parsepmcs.py</code> from <a href='https://svn.apache.org/repos/asf/infrastructure/site-tools/trunk/projects/data_files/'>PMC DOAP files</a></li>
<li><a href="projects.json"><code>projects.json</code></a> imported with <code>import/rdfparse.py</code> from <a href='https://svn.apache.org/repos/asf/infrastructure/site-tools/trunk/projects/files.xml'>projects' DOAP files</a></li>
<li><a href="releases.json"><code>releases.json</code></a> generated by <code>cronjobs/parsereleases.py</code> from <a href="http://www.apache.org/dist/">http://www.apache.org/dist/</a></li>