You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kibble.apache.org by hu...@apache.org on 2017/10/22 21:31:34 UTC

[kibble] branch master updated (c67ab8c -> 40fd867)

This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git.


    from c67ab8c  even as an array, this needs to be of type string
     new 5258029  make this actually work
     new 728fa5b  add topN authors API
     new bb37bb6  add topN topics API
     new 9fafa0d  tie the widgets into the mail page
     new 40fd867  regen API specs

The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 api/pages/mail/timeseries.py                       | 36 +++++++---
 .../{issue/closers.py => mail/top-authors.py}      | 45 +++++-------
 api/pages/{issue/top.py => mail/top-topics.py}     | 51 ++++++--------
 api/yaml/openapi.yaml                              | 80 ++++++++++++++++++++++
 api/yaml/widgets.yaml                              | 46 +++----------
 5 files changed, 158 insertions(+), 100 deletions(-)
 copy api/pages/{issue/closers.py => mail/top-authors.py} (84%)
 copy api/pages/{issue/top.py => mail/top-topics.py} (78%)

-- 
To stop receiving notification emails like this one, please contact
['"commits@kibble.apache.org" <co...@kibble.apache.org>'].

[kibble] 02/05: add topN authors API

Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git

commit 728fa5b6390248ba3318fb0d5aadaf7bd046efe1
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Oct 22 23:30:52 2017 +0200

    add topN authors API
---
 api/pages/mail/top-authors.py | 173 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 173 insertions(+)

diff --git a/api/pages/mail/top-authors.py b/api/pages/mail/top-authors.py
new file mode 100644
index 0000000..55879ef
--- /dev/null
+++ b/api/pages/mail/top-authors.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################
+# OPENAPI-URI: /api/mail/top-authors
+########################################################################
+# get:
+#   responses:
+#     '200':
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/CommitterList'
+#       description: 200 Response
+#     default:
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/Error'
+#       description: unexpected error
+#   security:
+#   - cookieAuth: []
+#   summary: Shows the top N of email authors
+# post:
+#   requestBody:
+#     content:
+#       application/json:
+#         schema:
+#           $ref: '#/components/schemas/defaultWidgetArgs'
+#   responses:
+#     '200':
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/CommitterList'
+#       description: 200 Response
+#     default:
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/Error'
+#       description: unexpected error
+#   security:
+#   - cookieAuth: []
+#   summary: Shows the top N of email authors
+# 
+########################################################################
+
+
+
+
+
+"""
+This is the TopN committers list renderer for Kibble
+"""
+
+import json
+import time
+import hashlib
+
+def run(API, environ, indata, session):
+    
+    # We need to be logged in for this!
+    if not session.user:
+        raise API.exception(403, "You must be logged in to use this API endpoint! %s")
+    
+    now = time.time()
+    
+    # First, fetch the view if we have such a thing enabled
+    viewList = []
+    if indata.get('view'):
+        viewList = session.getView(indata.get('view'))
+    if indata.get('subfilter'):
+        viewList = session.subFilter(indata.get('subfilter'), view = viewList) 
+    
+    
+    dateTo = indata.get('to', int(time.time()))
+    dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
+    
+    interval = indata.get('interval', 'month')
+    
+    
+    ####################################################################
+    ####################################################################
+    dOrg = session.user['defaultOrganisation'] or "apache"
+    query = {
+                'query': {
+                    'bool': {
+                        'must': [
+                            {'range':
+                                {
+                                    'ts': {
+                                        'from': dateFrom,
+                                        'to': dateTo
+                                    }
+                                }
+                            },
+                            {
+                                'term': {
+                                    'organisation': dOrg
+                                }
+                            }
+                        ]
+                    }
+                }
+            }
+    # Source-specific or view-specific??
+    if indata.get('source'):
+        query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
+    elif viewList:
+        query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
+    
+    # Get top 25 committers this period
+    query['aggs'] = {
+            'authors': {
+                'terms': {
+                    'field': 'sender',
+                    'size': 25
+                }
+            }            
+        }
+    res = session.DB.ES.search(
+            index=session.DB.dbname,
+            doc_type="email",
+            size = 0,
+            body = query
+        )
+
+    people = {}
+    for bucket in res['aggregations']['authors']['buckets']:
+        email = bucket['key']
+        count = bucket['doc_count']
+        sha = hashlib.sha1( ("%s%s" % (dOrg, email)).encode('utf-8') ).hexdigest()
+        if session.DB.ES.exists(index=session.DB.dbname,doc_type="person",id = sha):
+            pres = session.DB.ES.get(
+                index=session.DB.dbname,
+                doc_type="person",
+                id = sha
+                )
+            person = pres['_source']
+            person['name'] = person.get('name', 'unknown')
+            people[email] = person
+            people[email]['gravatar'] = hashlib.md5(person.get('email', 'unknown').encode('utf-8')).hexdigest()
+            people[email]['count'] = count
+    
+    topN = []
+    for email, person in people.items():
+        topN.append(person)
+    topN = sorted(topN, key = lambda x: x['count'], reverse = True)
+        
+    JSON_OUT = {
+        'topN': {
+            'denoter': 'emails',
+            'items': topN
+        },
+        'sorted': people,
+        'okay': True,
+        'responseTime': time.time() - now
+    }
+    yield json.dumps(JSON_OUT)

-- 
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.

[kibble] 03/05: add topN topics API

Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git

commit bb37bb6be0cb3ae41aba30f392d9f0a7448419ce
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Oct 22 23:31:08 2017 +0200

    add topN topics API
---
 api/pages/mail/top-topics.py | 153 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 153 insertions(+)

diff --git a/api/pages/mail/top-topics.py b/api/pages/mail/top-topics.py
new file mode 100644
index 0000000..9acda6c
--- /dev/null
+++ b/api/pages/mail/top-topics.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################
+# OPENAPI-URI: /api/mail/top-topics
+########################################################################
+# get:
+#   responses:
+#     '200':
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/CommitterList'
+#       description: 200 Response
+#     default:
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/Error'
+#       description: unexpected error
+#   security:
+#   - cookieAuth: []
+#   summary: Shows the top N of email authors
+# post:
+#   requestBody:
+#     content:
+#       application/json:
+#         schema:
+#           $ref: '#/components/schemas/defaultWidgetArgs'
+#   responses:
+#     '200':
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/CommitterList'
+#       description: 200 Response
+#     default:
+#       content:
+#         application/json:
+#           schema:
+#             $ref: '#/components/schemas/Error'
+#       description: unexpected error
+#   security:
+#   - cookieAuth: []
+#   summary: Shows the top N of email authors
+# 
+########################################################################
+
+
+
+
+
+"""
+This is the TopN committers list renderer for Kibble
+"""
+
+import json
+import time
+import hashlib
+
+def run(API, environ, indata, session):
+    
+    # We need to be logged in for this!
+    if not session.user:
+        raise API.exception(403, "You must be logged in to use this API endpoint! %s")
+    
+    now = time.time()
+    
+    # First, fetch the view if we have such a thing enabled
+    viewList = []
+    if indata.get('view'):
+        viewList = session.getView(indata.get('view'))
+    if indata.get('subfilter'):
+        viewList = session.subFilter(indata.get('subfilter'), view = viewList) 
+    
+    
+    dateTo = indata.get('to', int(time.time()))
+    dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
+    
+    interval = indata.get('interval', 'month')
+    
+    
+    ####################################################################
+    ####################################################################
+    dOrg = session.user['defaultOrganisation'] or "apache"
+    query = {
+                'query': {
+                    'bool': {
+                        'must': [
+                            {'range':
+                                {
+                                    'ts': {
+                                        'from': dateFrom,
+                                        'to': dateTo
+                                    }
+                                }
+                            },
+                            {
+                                'term': {
+                                    'organisation': dOrg
+                                }
+                            }
+                        ]
+                    }
+                },
+                'sort': [{
+                    'emails': 'desc'
+                }]
+            }
+    # Source-specific or view-specific??
+    if indata.get('source'):
+        query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
+    elif viewList:
+        query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
+    
+    res = session.DB.ES.search(
+            index=session.DB.dbname,
+            doc_type="mailtop",
+            size = 25,
+            body = query
+        )
+
+    topN = []
+    for bucket in res['hits']['hits']:
+        topN.append( {
+            'source': bucket['_source']['sourceURL'],
+            'name': bucket['_source']['subject'],
+            'count': bucket['_source']['emails']
+        })    
+        
+    JSON_OUT = {
+        'topN': {
+            'denoter': 'emails',
+            'items': topN,
+            'icon': 'envelope'
+        },
+        'okay': True,
+        'responseTime': time.time() - now
+    }
+    yield json.dumps(JSON_OUT)

-- 
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.

[kibble] 01/05: make this actually work

Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git

commit 5258029b58c2cd38de396fd3875963fbdceb6764
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Oct 22 23:30:27 2017 +0200

    make this actually work
---
 api/pages/mail/timeseries.py | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/api/pages/mail/timeseries.py b/api/pages/mail/timeseries.py
index b89b43a..75eb419 100644
--- a/api/pages/mail/timeseries.py
+++ b/api/pages/mail/timeseries.py
@@ -108,9 +108,9 @@ def run(API, environ, indata, session):
                         'must': [
                             {'range':
                                 {
-                                    'ts': {
-                                        'from': dateFrom,
-                                        'to': dateTo
+                                    'date': {
+                                        'from': time.strftime("%Y/%m/%d 0:0:0", time.gmtime(dateFrom)),
+                                        'to': time.strftime("%Y/%m/%d 23:59:59", time.gmtime(dateTo))
                                     }
                                 }
                             },
@@ -134,27 +134,45 @@ def run(API, environ, indata, session):
     
     # Get number of committers, this period
     query['aggs'] = {
-            'commits': {
+            'timeseries': {
                 'date_histogram': {
                     'field': 'date',
                     'interval': interval
-                }                
+                },
+                'aggs': {
+                    'email': {
+                        'sum': {
+                            'field': 'emails'
+                        }
+                    },
+                    'topics': {
+                        'sum': {
+                            'field': 'topics'
+                        }
+                    },
+                    'authors': {
+                        'sum': {
+                            'field': 'authors'
+                        }
+                    }
+                }
             }
         }
     res = session.DB.ES.search(
             index=session.DB.dbname,
-            doc_type="email",
+            doc_type="mailstats",
             size = 0,
             body = query
         )
     
     timeseries = []
-    for bucket in res['aggregations']['commits']['buckets']:
+    for bucket in res['aggregations']['timeseries']['buckets']:
         ts = int(bucket['key'] / 1000)
-        count = bucket['doc_count']
         timeseries.append({
             'date': ts,
-            'email': count
+            'emails': bucket['email']['value'],
+            'topics': bucket['topics']['value'],
+            'authors': bucket['authors']['value']
         })
     
     JSON_OUT = {

-- 
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.

[kibble] 05/05: regen API specs

Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git

commit 40fd86700b2090da7d5c6b57bef93fd6ea393d52
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Oct 22 23:31:26 2017 +0200

    regen API specs
---
 api/yaml/openapi.yaml | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/api/yaml/openapi.yaml b/api/yaml/openapi.yaml
index ac7a480..eb68027 100644
--- a/api/yaml/openapi.yaml
+++ b/api/yaml/openapi.yaml
@@ -1789,6 +1789,86 @@ paths:
       security:
       - cookieAuth: []
       summary: Shows email sent over time
+  /api/mail/top-authors:
+    get:
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CommitterList'
+          description: 200 Response
+        default:
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+          description: unexpected error
+      security:
+      - cookieAuth: []
+      summary: Shows the top N of email authors
+    post:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/defaultWidgetArgs'
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CommitterList'
+          description: 200 Response
+        default:
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+          description: unexpected error
+      security:
+      - cookieAuth: []
+      summary: Shows the top N of email authors
+  /api/mail/top-topics:
+    get:
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CommitterList'
+          description: 200 Response
+        default:
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+          description: unexpected error
+      security:
+      - cookieAuth: []
+      summary: Shows the top N of email authors
+    post:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/defaultWidgetArgs'
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CommitterList'
+          description: 200 Response
+        default:
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+          description: unexpected error
+      security:
+      - cookieAuth: []
+      summary: Shows the top N of email authors
   /api/mail/trends:
     get:
       responses:

-- 
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.

[kibble] 04/05: tie the widgets into the mail page

Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git

commit 9fafa0d0fe3671324a08d755b5e11b3602f83048
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Oct 22 23:31:18 2017 +0200

    tie the widgets into the mail page
---
 api/yaml/widgets.yaml | 46 +++++++++++-----------------------------------
 1 file changed, 11 insertions(+), 35 deletions(-)

diff --git a/api/yaml/widgets.yaml b/api/yaml/widgets.yaml
index 55d8107..badf4e8 100644
--- a/api/yaml/widgets.yaml
+++ b/api/yaml/widgets.yaml
@@ -382,54 +382,30 @@ widgets:
                         name:   "Mailing list trends"
                         blocks: 12
             -
-                name: "Issue tracker timeline"
+                name: "Email timeline"
                 children:
                     -
                         type:       "line"
-                        datatype:   "issue"
-                        blocks:     4
-                        source:     "issue/issues"
-                        name:       "Issues opened/closed"
-                        target:     ""
-                    -
-                        type:       "line"
-                        datatype:   "issue"
-                        blocks:     4
-                        source:     "issue/actors"
-                        name:       "Openers/Closers of issues"
-                        target:     ""
-                    -
-                        type:       "line"
-                        datatype:   "issue"
-                        blocks:     4
-                        source:     "issue/age"
-                        name:       "Issue age"
-                        target:     ""
-            -
-                name: "Issues and handlers"
-                children:
-                    -
-                        type:       "top5"
-                        datatype:   "issue"
+                        datatype:   "mail"
                         blocks:     4
-                        source:     "issue/openers"
-                        name:       "Top 5 issue openers"
+                        source:     "mail/timeseries"
+                        name:       "Email, topics, authors over time"
                         target:     ""
                     -
                         type:       "top5"
-                        datatype:   "issue"
+                        datatype:   "mail"
                         blocks:     4
-                        source:     "issue/closers"
-                        name:       "Top 5 issue closers"
+                        source:     "mail/top-authors"
+                        name:       "Most active authors"
                         target:     ""
                     -
                         type:       "top5"
-                        datatype:   "issue"
+                        datatype:   "mail"
                         blocks:     4
-                        source:     "issue/top"
-                        name:       "Most active issues"
+                        source:     "mail/top-topics"
+                        name:       "Most active topics"
                         target:     ""
-
+            
 
 ###### RELATIONSHIPS
 

-- 
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.