You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kibble.apache.org by hu...@apache.org on 2017/10/23 17:04:54 UTC
[kibble-scanners] branch master updated (e38e48c -> 25856e3)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git.
from e38e48c note that ForkManager is required here
new 1bdcb9f kibblebit shortcut for dbname, notify when pushing stragglers
new b932fc3 add a small plugin for tone analysis
new 7851561 add ponymail tone analysis
new 25856e3 ignore robits
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/plugins/brokers/kibbleES.py | 2 +
src/plugins/scanners/__init__.py | 1 +
src/plugins/scanners/ponymail-tone.py | 107 ++++++++++++++++++++++++++++++++++
src/plugins/utils/tone.py | 58 ++++++++++++++++++
4 files changed, 168 insertions(+)
create mode 100644 src/plugins/scanners/ponymail-tone.py
create mode 100644 src/plugins/utils/tone.py
--
To stop receiving notification emails like this one, please contact
['"commits@kibble.apache.org" <co...@kibble.apache.org>'].
[kibble-scanners] 04/04: ignore robits
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git
commit 25856e36d69aab3acd3670df76be7713bac516e7
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Oct 23 19:04:31 2017 +0200
ignore robits
---
src/plugins/scanners/ponymail-tone.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/plugins/scanners/ponymail-tone.py b/src/plugins/scanners/ponymail-tone.py
index d42e505..e3c96c1 100644
--- a/src/plugins/scanners/ponymail-tone.py
+++ b/src/plugins/scanners/ponymail-tone.py
@@ -95,7 +95,7 @@ def scan(KibbleBit, source):
rv = requests.get(emlurl).json()
body = rv['body']
KibbleBit.pprint("analyzing email")
- if 'mood' not in eml:
+ if 'mood' not in eml and not re.search(ROBITS, eml['sender']):
mood = plugins.utils.tone.getTone(KibbleBit, body)
eml['mood'] = mood
hm = [0,'unknown']
--
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.
[kibble-scanners] 03/04: add ponymail tone analysis
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git
commit 7851561856154dd32435d3c12cad024799444d81
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Oct 23 19:03:35 2017 +0200
add ponymail tone analysis
Only enabled it watson configured.
set to only grab the last 50 emails, so as to not
abuse the service. WIP!
---
src/plugins/scanners/__init__.py | 1 +
src/plugins/scanners/ponymail-tone.py | 107 ++++++++++++++++++++++++++++++++++
2 files changed, 108 insertions(+)
diff --git a/src/plugins/scanners/__init__.py b/src/plugins/scanners/__init__.py
index 7a6bc94..f0ace47 100644
--- a/src/plugins/scanners/__init__.py
+++ b/src/plugins/scanners/__init__.py
@@ -30,6 +30,7 @@ __all__ = [
'git-evolution',
'jira',
'ponymail',
+ 'ponymail-tone',
'pipermail',
'github-issues',
'bugzilla',
diff --git a/src/plugins/scanners/ponymail-tone.py b/src/plugins/scanners/ponymail-tone.py
new file mode 100644
index 0000000..d42e505
--- /dev/null
+++ b/src/plugins/scanners/ponymail-tone.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+ #the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This is a Kibble scanner plugin for Apache Pony Mail sources.
+"""
+import requests
+import json
+import time
+import re
+import hashlib
+import sys
+import datetime
+import plugins.utils.jsonapi
+import plugins.utils.tone
+
+title = "Tone/Mood Scanner plugin for Apache Pony Mail"
+version = "0.1.0"
+ROBITS = r"(git|gerrit|jenkins|hudson|builds)@"
+
+def accepts(source):
+ """ Test if source matches a Pony Mail archive """
+ # If the source equals the plugin name, assume a yes
+ if source['type'] == 'ponymail':
+ return True
+
+ # If it's of type 'mail', check the URL
+ if source['type'] == 'mail':
+ if re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source['sourceURL']):
+ return True
+
+ # Default to not recognizing the source
+ return False
+
+
+def scan(KibbleBit, source):
+ # Validate URL first
+ url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source['sourceURL'])
+ if not url:
+ KibbleBit.pprint("Malformed or invalid Pony Mail URL passed to scanner: %s" % source['sourceURL'])
+ source['steps']['mail'] = {
+ 'time': time.time(),
+ 'status': 'Could not parse Pony Mail URL!',
+ 'running': False,
+ 'good': False
+ }
+ KibbleBit.updateSource(source)
+ return
+ if not 'watson' in KibbleBit.config:
+ KibbleBit.pprint("No Watson/BlueMix creds configured, skipping tone analyzer")
+ return
+
+ rootURL = re.sub(r"list.html.+", "", source['sourceURL'])
+ query = {
+ 'query': {
+ 'bool': {
+ 'must': [
+ {
+ 'term': {
+ 'sourceID': source['sourceID']
+ }
+ }
+ ]
+ }
+ },
+ 'sort': [{'ts': 'desc'}]
+ }
+
+ # Get an initial count of commits
+ res = KibbleBit.broker.DB.search(
+ index=KibbleBit.dbname,
+ doc_type="email",
+ body = query,
+ size = 50
+ )
+
+ for hit in res['hits']['hits']:
+ eml = hit['_source']
+ emlurl = "%s/api/email.lua?id=%s" % (rootURL, eml['id'])
+ KibbleBit.pprint("Fetching %s" % emlurl)
+ rv = requests.get(emlurl).json()
+ body = rv['body']
+ KibbleBit.pprint("analyzing email")
+ if 'mood' not in eml:
+ mood = plugins.utils.tone.getTone(KibbleBit, body)
+ eml['mood'] = mood
+ hm = [0,'unknown']
+ for m, s in mood.items():
+ if s > hm[0]:
+ hm = [s,m]
+ print("Likeliest overall mood: %s" % hm[1])
+ KibbleBit.index('email', hit['_id'], eml)
+ KibbleBit.pprint("Done with tone analysis")
--
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.
[kibble-scanners] 01/04: kibblebit shortcut for dbname,
notify when pushing stragglers
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git
commit 1bdcb9fcb0b9820bee9dc8be577b329f29c3fa3f
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Oct 23 19:02:02 2017 +0200
kibblebit shortcut for dbname, notify when pushing stragglers
---
src/plugins/brokers/kibbleES.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/plugins/brokers/kibbleES.py b/src/plugins/brokers/kibbleES.py
index 8da1478..272f61d 100644
--- a/src/plugins/brokers/kibbleES.py
+++ b/src/plugins/brokers/kibbleES.py
@@ -42,10 +42,12 @@ class KibbleBit:
self.queueMax = 1000 # Entries to keep before bulk pushing
self.pluginname = ""
self.tid = tid
+ self.dbname = self.broker.config['elasticsearch']['database']
def __del__(self):
""" On unload/delete, push the last chunks of data to ES """
if self.json_queue:
+ print("Pushing stragglers")
self.bulk()
def pprint(self, string, err = False):
--
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.
[kibble-scanners] 02/04: add a small plugin for tone analysis
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git
commit b932fc3b9cb5071271058828ae0703f7cfb8d026
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Oct 23 19:02:23 2017 +0200
add a small plugin for tone analysis
---
src/plugins/utils/tone.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 58 insertions(+)
diff --git a/src/plugins/utils/tone.py b/src/plugins/utils/tone.py
new file mode 100644
index 0000000..85d9cad
--- /dev/null
+++ b/src/plugins/utils/tone.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+ #the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This is an experimental tone analyzer plugin for using Watson/BlueMix for
+analyzing the mood of email on a list. This requires a Watson account
+and a watson section in config.yaml, as such:
+
+watson:
+ username: $user
+ password: $pass
+ api: https://$something.watsonplatform.net/tone-analyzer/api
+
+Currently only pony mail is supported. more to come.
+"""
+
+import time
+import datetime
+import re
+import json
+import hashlib
+import requests
+import json
+
+def getTone(KibbleBit, body):
+ if 'watson' in KibbleBit.config:
+ headers = {
+ 'Content-Type': 'application/json'
+ }
+ js = {
+ 'text': body
+ }
+ rv = requests.post(
+ "%s/v3/tone?version=2017-09-21&sentences=false" % KibbleBit.config['watson']['api'],
+ headers = headers,
+ data = json.dumps(js),
+ auth = (KibbleBit.config['watson']['username'], KibbleBit.config['watson']['password'])
+ )
+ mood = {}
+ jsout = rv.json()
+ for tone in jsout['document_tone']['tones']:
+ mood[tone['tone_id']] = tone['score']
+ return mood
+
--
To stop receiving notification emails like this one, please contact
"commits@kibble.apache.org" <co...@kibble.apache.org>.