You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kibble.apache.org by tu...@apache.org on 2020/12/13 13:53:48 UTC
[kibble] branch main updated: Improve code quality of scanners
(#120)
This is an automated email from the ASF dual-hosted git repository.
turbaszek pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/kibble.git
The following commit(s) were added to refs/heads/main by this push:
new f8d731c Improve code quality of scanners (#120)
f8d731c is described below
commit f8d731cedf51fcd5a19e05c8ed8db919f1c7838a
Author: Tomek Urbaszek <tu...@gmail.com>
AuthorDate: Sun Dec 13 13:53:37 2020 +0000
Improve code quality of scanners (#120)
This PR replaces cameleCase with snake_case, removes some unused
variables and adds some that in some cases may result in run time error.
---
kibble/scanners/brokers/kibbleES.py | 2 +-
kibble/scanners/scanners/bugzilla.py | 98 +++++++++--------
kibble/scanners/scanners/buildbot.py | 58 +++++-----
kibble/scanners/scanners/discourse.py | 62 +++++------
kibble/scanners/scanners/gerrit.py | 26 ++---
kibble/scanners/scanners/git-census.py | 27 +++--
kibble/scanners/scanners/git-evolution.py | 40 +++----
kibble/scanners/scanners/git-sloc.py | 14 +--
kibble/scanners/scanners/git-sync.py | 36 +++---
kibble/scanners/scanners/github-issues.py | 52 ++++-----
kibble/scanners/scanners/github-stats.py | 24 ++--
kibble/scanners/scanners/jenkins.py | 93 ++++++++--------
kibble/scanners/scanners/jira.py | 176 +++++++++++++++---------------
kibble/scanners/scanners/pipermail.py | 56 +++++-----
kibble/scanners/scanners/ponymail-kpe.py | 43 ++++----
kibble/scanners/scanners/ponymail-tone.py | 50 ++++-----
kibble/scanners/scanners/ponymail.py | 83 +++++++-------
kibble/scanners/scanners/travis.py | 158 +++++++++++++--------------
kibble/scanners/scanners/twitter.py | 36 +++---
19 files changed, 561 insertions(+), 573 deletions(-)
diff --git a/kibble/scanners/brokers/kibbleES.py b/kibble/scanners/brokers/kibbleES.py
index 4d372f6..5e6eb0d 100644
--- a/kibble/scanners/brokers/kibbleES.py
+++ b/kibble/scanners/brokers/kibbleES.py
@@ -141,7 +141,7 @@ class KibbleBit:
else:
print(line)
- def updateSource(self, source):
+ def update_source(self, source):
""" Updates a source document, usually with a status update """
self.broker.DB.index(
index=self.broker.config["elasticsearch"]["database"],
diff --git a/kibble/scanners/scanners/bugzilla.py b/kibble/scanners/scanners/bugzilla.py
index e1f1bfb..fa1833c 100644
--- a/kibble/scanners/scanners/bugzilla.py
+++ b/kibble/scanners/scanners/bugzilla.py
@@ -43,7 +43,7 @@ def accepts(source):
return False
-def getTime(string):
+def get_time(string):
return time.mktime(
time.strptime(re.sub(r"[zZ]", "", str(string)), "%Y-%m-%dT%H:%M:%S")
)
@@ -81,7 +81,7 @@ def moved(js):
return False
-def wasclosed(js):
+def was_closed(js):
if "changelog" in js:
cjs = js["changelog"]["histories"]
for citem in cjs:
@@ -118,7 +118,7 @@ def pchange(js):
return False
-def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
+def scan_ticket(bug, kibble_bit, source, open_tickets, u, dom):
try:
key = bug["id"]
dhash = hashlib.sha224(
@@ -126,24 +126,24 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
"ascii", errors="replace"
)
).hexdigest()
- found = KibbleBit.exists("issue", dhash)
+ found = kibble_bit.exists("issue", dhash)
parseIt = False
if not found:
parseIt = True
else:
- ticket = KibbleBit.get("issue", dhash)
- if ticket["status"] == "closed" and key in openTickets:
- KibbleBit.pprint("Ticket was reopened, reparsing")
+ ticket = kibble_bit.get("issue", dhash)
+ if ticket["status"] == "closed" and key in open_tickets:
+ kibble_bit.pprint("Ticket was reopened, reparsing")
parseIt = True
- elif ticket["status"] == "open" and not key in openTickets:
- KibbleBit.pprint("Ticket was recently closed, parsing it")
+ elif ticket["status"] == "open" and not key in open_tickets:
+ kibble_bit.pprint("Ticket was recently closed, parsing it")
parseIt = True
else:
pass
# print("Ticket hasn't changed, ignoring...")
if parseIt:
- KibbleBit.pprint("Parsing data from BugZilla for #%s" % key)
+ kibble_bit.pprint("Parsing data from BugZilla for #%s" % key)
params = {"ids": [int(key)], "limit": 0}
if (
@@ -163,12 +163,12 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
js = js["result"]["bugs"][0]
creator = {"name": bug["creator"], "email": js["creator"]}
closer = {}
- cd = getTime(js["creation_time"])
+ cd = get_time(js["creation_time"])
rd = None
status = "open"
if js["status"] in ["CLOSED", "RESOLVED"]:
status = "closed"
- KibbleBit.pprint("%s was closed, finding out who did that" % key)
+ kibble_bit.pprint("%s was closed, finding out who did that" % key)
ticketsURL = "%s?method=Bug.history¶ms=[%s]" % (
u,
urllib.parse.quote(json.dumps(params)),
@@ -182,10 +182,10 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
and "added" in change
and change["added"] in ["CLOSED", "RESOLVED"]
):
- rd = getTime(item["when"])
+ rd = get_time(item["when"])
closer = {"name": item["who"], "email": item["who"]}
break
- KibbleBit.pprint("Counting comments for %s..." % key)
+ kibble_bit.pprint("Counting comments for %s..." % key)
ticketsURL = "%s?method=Bug.comments¶ms=[%s]" % (
u,
urllib.parse.quote(json.dumps(params)),
@@ -202,7 +202,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
"ascii", errors="replace"
)
).hexdigest()
- found = KibbleBit.exists("person", pid)
+ found = kibble_bit.exists("person", pid)
if not found:
params["names"] = [closer["email"]]
ticketsURL = "%s?method=User.get¶ms=[%s]" % (
@@ -213,7 +213,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
try:
ujs = jsonapi.get(ticketsURL)
displayName = ujs["result"]["users"][0]["real_name"]
- except:
+ except: # pylint: disable=bare-except
displayName = closer["email"]
if displayName and len(displayName) > 0:
# Add to people db
@@ -225,7 +225,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
"id": pid,
}
# print("Updating person DB for closer: %s (%s)" % (displayName, closerEmail))
- KibbleBit.index("person", pid, jsp)
+ kibble_bit.index("person", pid, jsp)
if creator:
pid = hashlib.sha1(
@@ -233,7 +233,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
"ascii", errors="replace"
)
).hexdigest()
- found = KibbleBit.exists("person", pid)
+ found = kibble_bit.exists("person", pid)
if not found:
if not creator["name"]:
params["names"] = [creator["email"]]
@@ -244,7 +244,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
try:
ujs = jsonapi.get(ticketsURL)
creator["name"] = ujs["result"]["users"][0]["real_name"]
- except:
+ except: # pylint: disable=bare-except
creator["name"] = creator["email"]
if creator["name"] and len(creator["name"]) > 0:
# Add to people db
@@ -255,7 +255,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
"organisation": source["organisation"],
"id": pid,
}
- KibbleBit.index("person", pid, jsp)
+ kibble_bit.index("person", pid, jsp)
jso = {
"id": dhash,
@@ -280,17 +280,17 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom):
"comments": comments,
"title": title,
}
- KibbleBit.append("issue", jso)
+ kibble_bit.append("issue", jso)
time.sleep(0.5) # BugZilla is notoriously slow. Maybe remove this later
return True
except Exception as err:
- KibbleBit.pprint(err)
+ kibble_bit.pprint(err)
return False
-class bzThread(Thread):
+class BzThread(Thread):
def __init__(self, KibbleBit, source, block, pt, ot, u, dom):
- super(bzThread, self).__init__()
+ super(BzThread, self).__init__()
self.KibbleBit = KibbleBit
self.source = source
self.block = block
@@ -300,9 +300,9 @@ class bzThread(Thread):
self.dom = dom
def run(self):
- badOnes = 0
+ bad_ones = 0
- while len(self.pendingTickets) > 0 and badOnes <= 50:
+ while len(self.pendingTickets) > 0 and bad_ones <= 50:
if len(self.pendingTickets) % 10 == 0:
self.KibbleBit.pprint(
"%u elements left to count" % len(self.pendingTickets)
@@ -317,12 +317,12 @@ class bzThread(Thread):
self.block.release()
return
self.block.release()
- if not scanTicket(
+ if not scan_ticket(
rl, self.KibbleBit, self.source, self.openTickets, self.u, self.dom
):
self.KibbleBit.pprint("Ticket %s seems broken, skipping" % rl["id"])
- badOnes += 1
- if badOnes > 50:
+ bad_ones += 1
+ if bad_ones > 50:
self.KibbleBit.pprint("Too many errors, bailing!")
self.source["steps"]["issues"] = {
"time": time.time(),
@@ -331,13 +331,13 @@ class bzThread(Thread):
"running": False,
"good": False,
}
- self.KibbleBit.updateSource(self.source)
+ self.KibbleBit.update_source(self.source)
return
else:
- badOnes = 0
+ bad_ones = 0
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
url = source["sourceURL"]
source["steps"]["issues"] = {
@@ -346,7 +346,7 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
bz = re.match(r"(https?://\S+?)(/jsonrpc\.cgi)?[\s:?]+(.+)", url)
if bz:
@@ -357,8 +357,8 @@ def scan(KibbleBit, source):
and len(source["creds"]["username"]) > 0
):
creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"])
- pendingTickets = []
- openTickets = []
+ pending_tickets = []
+ open_tickets = []
# Get base URL, list and domain to parse
dom = bz.group(1)
@@ -404,20 +404,20 @@ def scan(KibbleBit, source):
"offset": 1,
}
- ticketsURL = "%s?method=Bug.search¶ms=[%s]" % (
+ tickets_url = "%s?method=Bug.search¶ms=[%s]" % (
u,
urllib.parse.quote(json.dumps(params)),
)
while True:
try:
- js = jsonapi.get(ticketsURL, auth=creds)
- except:
- KibbleBit.pprint("Couldn't fetch more tickets, bailing")
+ js = jsonapi.get(tickets_url, auth=creds)
+ except: # pylint: disable=bare-except
+ kibble_bit.pprint("Couldn't fetch more tickets, bailing")
break
if len(js["result"]["bugs"]) > 0:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"%s: Found %u tickets..."
% (
source["sourceURL"],
@@ -425,28 +425,30 @@ def scan(KibbleBit, source):
)
)
for bug in js["result"]["bugs"]:
- pendingTickets.append(bug)
+ pending_tickets.append(bug)
if not bug["status"] in ["RESOLVED", "CLOSED"]:
- openTickets.append(bug["id"])
+ open_tickets.append(bug["id"])
params["offset"] += 10000
- ticketsURL = "%s?method=Bug.search¶ms=[%s]" % (
+ tickets_url = "%s?method=Bug.search¶ms=[%s]" % (
u,
urllib.parse.quote(json.dumps(params)),
)
else:
- KibbleBit.pprint("No more tickets left to scan")
+ kibble_bit.pprint("No more tickets left to scan")
break
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Found %u open tickets, %u closed."
- % (len(openTickets), len(pendingTickets) - len(openTickets))
+ % (len(open_tickets), len(pending_tickets) - len(open_tickets))
)
block = Lock()
threads = []
# TODO: Fix this loop
for i in range(0, 4):
- t = bzThread(KibbleBit, source, block, pendingTickets, openTickets, u, dom)
+ t = BzThread(
+ kibble_bit, source, block, pending_tickets, open_tickets, u, dom
+ )
threads.append(t)
t.start()
@@ -460,4 +462,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/buildbot.py b/kibble/scanners/scanners/buildbot.py
index 190fb5e..6d2e18d 100644
--- a/kibble/scanners/scanners/buildbot.py
+++ b/kibble/scanners/scanners/buildbot.py
@@ -38,7 +38,7 @@ def accepts(source):
return False
-def scanJob(KibbleBit, source, job, creds):
+def scan_job(KibbleBit, source, job, creds):
""" Scans a single job for activity """
dhash = hashlib.sha224(
("%s-%s-%s" % (source["organisation"], source["sourceID"], job)).encode(
@@ -64,7 +64,7 @@ def scanJob(KibbleBit, source, job, creds):
builddoc = None
try:
builddoc = KibbleBit.get("ci_build", buildhash)
- except:
+ except: # pylint: disable=bare-except
pass
# If this build already completed, no need to parse it again
@@ -120,11 +120,11 @@ def scanJob(KibbleBit, source, job, creds):
return False
-class buildbotThread(threading.Thread):
+class BuildbotThread(threading.Thread):
""" Generic thread class for scheduling multiple scans at once """
def __init__(self, block, KibbleBit, source, creds, jobs):
- super(buildbotThread, self).__init__()
+ super(BuildbotThread, self).__init__()
self.block = block
self.KibbleBit = KibbleBit
self.creds = creds
@@ -132,8 +132,8 @@ class buildbotThread(threading.Thread):
self.jobs = jobs
def run(self):
- badOnes = 0
- while len(self.jobs) > 0 and badOnes <= 50:
+ bad_ones = 0
+ while len(self.jobs) > 0 and bad_ones <= 50:
self.block.acquire()
try:
job = self.jobs.pop(0)
@@ -144,10 +144,10 @@ class buildbotThread(threading.Thread):
self.block.release()
return
self.block.release()
- if not scanJob(self.KibbleBit, self.source, job, self.creds):
+ if not scan_job(self.KibbleBit, self.source, job, self.creds):
self.KibbleBit.pprint("[%s] This borked, trying another one" % job)
- badOnes += 1
- if badOnes > 100:
+ bad_ones += 1
+ if bad_ones > 100:
self.KibbleBit.pprint("Too many errors, bailing!")
self.source["steps"]["ci"] = {
"time": time.time(),
@@ -156,13 +156,13 @@ class buildbotThread(threading.Thread):
"running": False,
"good": False,
}
- self.KibbleBit.updateSource(self.source)
+ self.KibbleBit.update_source(self.source)
return
else:
- badOnes = 0
+ bad_ones = 0
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Simple URL check
buildbot = re.match(r"(https?://.+)", source["sourceURL"])
if buildbot:
@@ -173,16 +173,16 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
- KibbleBit.pprint("Parsing Buildbot activity at %s" % source["sourceURL"])
+ kibble_bit.pprint("Parsing Buildbot activity at %s" % source["sourceURL"])
source["steps"]["ci"] = {
"time": time.time(),
"status": "Downloading changeset",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Buildbot may neeed credentials
creds = None
@@ -195,9 +195,9 @@ def scan(KibbleBit, source):
creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"])
# Get the job list
- sURL = source["sourceURL"]
- KibbleBit.pprint("Getting job list...")
- builders = jsonapi.get("%s/json/builders" % sURL, auth=creds)
+ s_url = source["sourceURL"]
+ kibble_bit.pprint("Getting job list...")
+ builders = jsonapi.get("%s/json/builders" % s_url, auth=creds)
# Save queue snapshot
NOW = int(datetime.datetime.utcnow().timestamp())
@@ -211,8 +211,8 @@ def scan(KibbleBit, source):
# Scan queue items
blocked = 0
stuck = 0
- queueSize = 0
- actualQueueSize = 0
+ queue_size = 0
+ actual_queue_size = 0
building = 0
jobs = []
@@ -222,10 +222,10 @@ def scan(KibbleBit, source):
building += 1
if data.get("pendingBuilds", 0) > 0:
# All queued items, even offline builders
- actualQueueSize += data.get("pendingBuilds", 0)
+ actual_queue_size += data.get("pendingBuilds", 0)
# Only queues with an online builder (actually waiting stuff)
if data["state"] == "building":
- queueSize += data.get("pendingBuilds", 0)
+ queue_size += data.get("pendingBuilds", 0)
blocked += data.get("pendingBuilds", 0) # Blocked by running builds
# Stuck builds (iow no builder available)
if data["state"] == "offline":
@@ -236,7 +236,7 @@ def scan(KibbleBit, source):
"id": queuehash,
"date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(NOW)),
"time": NOW,
- "size": queueSize,
+ "size": queue_size,
"blocked": blocked,
"stuck": stuck,
"building": building,
@@ -246,15 +246,15 @@ def scan(KibbleBit, source):
"organisation": source["organisation"],
"upsert": True,
}
- KibbleBit.append("ci_queue", queuedoc)
+ kibble_bit.append("ci_queue", queuedoc)
- KibbleBit.pprint("Found %u builders in Buildbot" % len(jobs))
+ kibble_bit.pprint("Found %u builders in Buildbot" % len(jobs))
threads = []
block = threading.Lock()
- KibbleBit.pprint("Scanning jobs using 4 sub-threads")
+ kibble_bit.pprint("Scanning jobs using 4 sub-threads")
for i in range(0, 4):
- t = buildbotThread(block, KibbleBit, source, creds, jobs)
+ t = BuildbotThread(block, kibble_bit, source, creds, jobs)
threads.append(t)
t.start()
@@ -262,7 +262,7 @@ def scan(KibbleBit, source):
t.join()
# We're all done, yaay
- KibbleBit.pprint("Done scanning %s" % source["sourceURL"])
+ kibble_bit.pprint("Done scanning %s" % source["sourceURL"])
source["steps"]["ci"] = {
"time": time.time(),
@@ -271,4 +271,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/discourse.py b/kibble/scanners/scanners/discourse.py
index ee99dde..4e1d92f 100644
--- a/kibble/scanners/scanners/discourse.py
+++ b/kibble/scanners/scanners/discourse.py
@@ -39,11 +39,11 @@ def accepts(source):
return False
-def scanJob(KibbleBit, source, cat, creds):
+def scan_job(kibble_bit, source, cat, creds):
""" Scans a single discourse category for activity """
# Get $discourseURL/c/$catID
catURL = os.path.join(source["sourceURL"], "c/%s" % cat["id"])
- KibbleBit.pprint("Scanning Discourse category '%s' at %s" % (cat["slug"], catURL))
+ kibble_bit.pprint("Scanning Discourse category '%s' at %s" % (cat["slug"], catURL))
page = 0
allUsers = {}
@@ -91,8 +91,8 @@ def scanJob(KibbleBit, source, cat, creds):
# Store it (or, queue storage) unless it exists.
# We don't wanna override better data, so we check if
# it's there first.
- if not KibbleBit.exists("person", dhash):
- KibbleBit.append("person", userDoc)
+ if not kibble_bit.exists("person", dhash):
+ kibble_bit.append("person", userDoc)
# Now, for each topic, we'll store a topic document
for topic in catjson["topic_list"]["topics"]:
@@ -118,8 +118,8 @@ def scanJob(KibbleBit, source, cat, creds):
# Determine whether we should scan this topic or continue to the next one.
# We'll do this by seeing if the topic already exists and has no changes or not.
- if KibbleBit.exists("forum_topic", dhash):
- fdoc = KibbleBit.get("forum_topic", dhash)
+ if kibble_bit.exists("forum_topic", dhash):
+ fdoc = kibble_bit.get("forum_topic", dhash)
# If update in the old doc was >= current update timestamp, skip the topic
if fdoc["updated"] >= UpdatedDate:
continue
@@ -146,8 +146,8 @@ def scanJob(KibbleBit, source, cat, creds):
+ "/t/%s/%s" % (topic["slug"], topic["id"]),
}
- KibbleBit.append("forum_topic", topicdoc)
- KibbleBit.pprint("%s is new or changed, scanning" % topicdoc["url"])
+ kibble_bit.append("forum_topic", topicdoc)
+ kibble_bit.pprint("%s is new or changed, scanning" % topicdoc["url"])
# Now grab all the individual replies/posts
# Remember to not have it count as a visit!
@@ -157,7 +157,7 @@ def scanJob(KibbleBit, source, cat, creds):
posts = pjson["post_stream"]["posts"]
# For each post/reply, construct a forum_entry document
- KibbleBit.pprint("%s has %u posts" % (pURL, len(posts)))
+ kibble_bit.pprint("%s has %u posts" % (pURL, len(posts)))
for post in posts:
phash = hashlib.sha224(
(
@@ -199,7 +199,7 @@ def scanJob(KibbleBit, source, cat, creds):
allUsers[user["id"]] = userDoc
# Store it (or, queue storage)
- KibbleBit.append("person", userDoc)
+ kibble_bit.append("person", userDoc)
# Get post date
CreatedDate = datetime.datetime.strptime(
@@ -222,18 +222,18 @@ def scanJob(KibbleBit, source, cat, creds):
"text": post["cooked"],
"url": topicdoc["url"],
}
- KibbleBit.append("forum_post", pdoc)
+ kibble_bit.append("forum_post", pdoc)
else:
- KibbleBit.pprint("Fetching discourse data failed!")
+ kibble_bit.pprint("Fetching discourse data failed!")
return False
return True
-class discourseThread(threading.Thread):
+class DiscourseThread(threading.Thread):
""" Generic thread class for scheduling multiple scans at once """
def __init__(self, block, KibbleBit, source, creds, jobs):
- super(discourseThread, self).__init__()
+ super(DiscourseThread, self).__init__()
self.block = block
self.KibbleBit = KibbleBit
self.creds = creds
@@ -241,8 +241,8 @@ class discourseThread(threading.Thread):
self.jobs = jobs
def run(self):
- badOnes = 0
- while len(self.jobs) > 0 and badOnes <= 50:
+ bad_ones = 0
+ while len(self.jobs) > 0 and bad_ones <= 50:
self.block.acquire()
try:
job = self.jobs.pop(0)
@@ -253,12 +253,12 @@ class discourseThread(threading.Thread):
self.block.release()
return
self.block.release()
- if not scanJob(self.KibbleBit, self.source, job, self.creds):
+ if not scan_job(self.KibbleBit, self.source, job, self.creds):
self.KibbleBit.pprint(
"[%s] This borked, trying another one" % job["name"]
)
- badOnes += 1
- if badOnes > 10:
+ bad_ones += 1
+ if bad_ones > 10:
self.KibbleBit.pprint("Too many errors, bailing!")
self.source["steps"]["forum"] = {
"time": time.time(),
@@ -267,13 +267,13 @@ class discourseThread(threading.Thread):
"running": False,
"good": False,
}
- self.KibbleBit.updateSource(self.source)
+ self.KibbleBit.update_source(self.source)
return
else:
- badOnes = 0
+ bad_ones = 0
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Simple URL check
discourse = re.match(r"(https?://.+)", source["sourceURL"])
if discourse:
@@ -284,17 +284,17 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
pendingJobs = []
- KibbleBit.pprint("Parsing Discourse activity at %s" % source["sourceURL"])
+ kibble_bit.pprint("Parsing Discourse activity at %s" % source["sourceURL"])
source["steps"]["forum"] = {
"time": time.time(),
"status": "Downloading changeset",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Discourse may neeed credentials (if basic auth)
creds = None
@@ -308,20 +308,20 @@ def scan(KibbleBit, source):
# Get the list of categories
sURL = source["sourceURL"]
- KibbleBit.pprint("Getting categories...")
+ kibble_bit.pprint("Getting categories...")
catjs = jsonapi.get("%s/categories_and_latest" % sURL, auth=creds)
# Directly assign the category list as pending jobs queue, ezpz.
pendingJobs = catjs["category_list"]["categories"]
- KibbleBit.pprint("Found %u categories" % len(pendingJobs))
+ kibble_bit.pprint("Found %u categories" % len(pendingJobs))
# Now fire off 4 threads to parse the categories
threads = []
block = threading.Lock()
- KibbleBit.pprint("Scanning jobs using 4 sub-threads")
+ kibble_bit.pprint("Scanning jobs using 4 sub-threads")
for i in range(0, 4):
- t = discourseThread(block, KibbleBit, source, creds, pendingJobs)
+ t = DiscourseThread(block, kibble_bit, source, creds, pendingJobs)
threads.append(t)
t.start()
@@ -329,7 +329,7 @@ def scan(KibbleBit, source):
t.join()
# We're all done, yaay
- KibbleBit.pprint("Done scanning %s" % source["sourceURL"])
+ kibble_bit.pprint("Done scanning %s" % source["sourceURL"])
source["steps"]["forum"] = {
"time": time.time(),
@@ -338,4 +338,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/gerrit.py b/kibble/scanners/scanners/gerrit.py
index b82dab6..fa60dff 100644
--- a/kibble/scanners/scanners/gerrit.py
+++ b/kibble/scanners/scanners/gerrit.py
@@ -160,15 +160,15 @@ def make_person(repo, raw_person):
}
-def update_issue(KibbleBit, issue):
+def update_issue(kibble_bit, issue):
id = issue["id"]
- KibbleBit.pprint("Updating issue: " + id)
- KibbleBit.index("issue", id, issue)
+ kibble_bit.pprint("Updating issue: " + id)
+ kibble_bit.index("issue", id, issue)
-def update_person(KibbleBit, person):
- KibbleBit.pprint("Updating person: " + person["name"] + " - " + person["email"])
- KibbleBit.index("person", person["id"], {"doc": person, "doc_as_upsert": True})
+def update_person(kibble_bit, person):
+ kibble_bit.pprint("Updating person: " + person["name"] + " - " + person["email"])
+ kibble_bit.index("person", person["id"], {"doc": person, "doc_as_upsert": True})
def status_changed(stored_change, change):
@@ -177,14 +177,14 @@ def status_changed(stored_change, change):
return stored_change["status"] != change["status"]
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
source["steps"]["issues"] = {
"time": time.time(),
"status": "Analyzing Gerrit tickets...",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
url = source["sourceURL"]
# Try matching foo.bar/r/project/subfoo
@@ -217,8 +217,8 @@ def scan(KibbleBit, source):
dhash = make_hash(source, change)
stored_change = None
- if KibbleBit.exists("issue", dhash):
- stored_change = KibbleBit.get("issue", dhash)
+ if kibble_bit.exists("issue", dhash):
+ stored_change = kibble_bit.get("issue", dhash)
if not status_changed(stored_change, change):
# print("change %s seen already and status unchanged. Skipping." %
@@ -228,7 +228,7 @@ def scan(KibbleBit, source):
details = change_details(base_url, change)
issue_doc = make_issue(source, base_url, details)
- update_issue(KibbleBit, issue_doc)
+ update_issue(kibble_bit, issue_doc)
labels = details["labels"]
change_people = []
@@ -247,7 +247,7 @@ def scan(KibbleBit, source):
for person in change_people:
if "email" in person and person["email"] not in people:
people[person["email"]] = person
- update_person(KibbleBit, make_person(source, person))
+ update_person(kibble_bit, make_person(source, person))
except requests.HTTPError as e:
print(e)
@@ -258,4 +258,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/git-census.py b/kibble/scanners/scanners/git-census.py
index aa19a03..790ae49 100644
--- a/kibble/scanners/scanners/git-census.py
+++ b/kibble/scanners/scanners/git-census.py
@@ -38,7 +38,7 @@ def accepts(source):
return False
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
""" Conduct a census scan """
people = {}
idseries = {}
@@ -50,7 +50,7 @@ def scan(KibbleBit, source):
rid = source["sourceID"]
url = source["sourceURL"]
rootpath = "%s/%s/git" % (
- KibbleBit.config["scanner"]["scratchdir"],
+ kibble_bit.config["scanner"]["scratchdir"],
source["organisation"],
)
gpath = os.path.join(rootpath, rid)
@@ -63,9 +63,8 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
gname = rid
- inp = ""
modificationDates = {}
# Did we do a census before?
if "census" in source and source["census"] > 0:
@@ -92,7 +91,7 @@ def scan(KibbleBit, source):
inp = f.read()
f.close()
os.unlink(tmp.name)
- KibbleBit.pprint("Parsing log for %s (%s)..." % (rid, url))
+ kibble_bit.pprint("Parsing log for %s (%s)..." % (rid, url))
for m in re.finditer(
u":([a-f0-9]+)\|([^\r\n|]+)\|([^\r\n|]+)\|([^\r\n|]+)\|([^\r\n|]+)\|([\d+]+)\r?\n([^:]+?):",
inp,
@@ -145,7 +144,7 @@ def scan(KibbleBit, source):
if delete > 100000000:
delete = 0
if delete > 1000000 or insert > 1000000:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"gigantic diff for %s (%s), ignoring"
% (gpath, source["sourceURL"])
)
@@ -250,7 +249,7 @@ def scan(KibbleBit, source):
"vcs": "git",
"files_changed": filelist,
}
- KibbleBit.append(
+ kibble_bit.append(
"person",
{
"upsert": True,
@@ -265,7 +264,7 @@ def scan(KibbleBit, source):
).hexdigest(),
},
)
- KibbleBit.append(
+ kibble_bit.append(
"person",
{
"upsert": True,
@@ -280,11 +279,11 @@ def scan(KibbleBit, source):
).hexdigest(),
},
)
- KibbleBit.append("code_commit", js)
- KibbleBit.append("code_commit_unique", jsx)
+ kibble_bit.append("code_commit", js)
+ kibble_bit.append("code_commit_unique", jsx)
if True: # Do file changes?? Might wanna make this optional
- KibbleBit.pprint("Scanning file changes for %s" % source["sourceURL"])
+ kibble_bit.pprint("Scanning file changes for %s" % source["sourceURL"])
for filename in modificationDates:
fid = hashlib.sha1(
("%s/%s" % (source["sourceID"], filename)).encode(
@@ -310,11 +309,11 @@ def scan(KibbleBit, source):
time.gmtime(modificationDates[filename]["created"]),
),
}
- found = KibbleBit.exists("file_history", fid)
+ found = kibble_bit.exists("file_history", fid)
if found:
del jsfe["created"]
del jsfe["createdDate"]
- KibbleBit.append("file_history", jsfe)
+ kibble_bit.append("file_history", jsfe)
source["steps"]["census"] = {
"time": time.time(),
@@ -324,4 +323,4 @@ def scan(KibbleBit, source):
"good": True,
}
source["census"] = time.time()
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/git-evolution.py b/kibble/scanners/scanners/git-evolution.py
index 07a7ada..b533b4b 100644
--- a/kibble/scanners/scanners/git-evolution.py
+++ b/kibble/scanners/scanners/git-evolution.py
@@ -46,12 +46,12 @@ def get_first_ref(gpath):
% gpath,
shell=True,
)
- except:
+ except: # pylint: disable=bare-except
print("Could not get first ref, exiting!")
return None
-def acquire(KibbleBit, source):
+def acquire(kibble_bit, source):
source["steps"]["evolution"] = {
"time": time.time(),
"status": "Evolution scan started at "
@@ -59,10 +59,10 @@ def acquire(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
-def release(KibbleBit, source, status, exception=None, good=False):
+def release(kibble_bit, source, status, exception=None, good=False):
source["steps"]["evolution"] = {
"time": time.time(),
"status": status,
@@ -72,7 +72,7 @@ def release(KibbleBit, source, status, exception=None, good=False):
if exception:
source["steps"]["evolution"].update({"exception": exception})
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
def check_branch(gpath, date, branch):
@@ -82,7 +82,7 @@ def check_branch(gpath, date, branch):
shell=True,
)
return True
- except:
+ except: # pylint: disable=bare-except
return False
@@ -116,7 +116,7 @@ def find_branch(date, gpath):
stderr=subprocess.DEVNULL,
)
return "master"
- except:
+ except: # pylint: disable=bare-except
os.chdir(gpath)
try:
return (
@@ -129,22 +129,22 @@ def find_branch(date, gpath):
.strip()
.strip("* ")
)
- except:
+ except: # pylint: disable=bare-except
# print("meh! no branch")
return None
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
rid = source["sourceID"]
rootpath = "%s/%s/git" % (
- KibbleBit.config["scanner"]["scratchdir"],
+ kibble_bit.config["scanner"]["scratchdir"],
source["organisation"],
)
gpath = os.path.join(rootpath, rid)
gname = source["sourceID"]
- KibbleBit.pprint("Doing evolution scan of %s" % gname)
+ kibble_bit.pprint("Doing evolution scan of %s" % gname)
inp = get_first_ref(gpath)
if inp:
@@ -158,13 +158,13 @@ def scan(KibbleBit, source):
rid = source["sourceID"]
url = source["sourceURL"]
rootpath = "%s/%s/git" % (
- KibbleBit.config["scanner"]["scratchdir"],
+ kibble_bit.config["scanner"]["scratchdir"],
source["organisation"],
)
gpath = os.path.join(rootpath, rid)
if source["steps"]["sync"]["good"] and os.path.exists(gpath):
- acquire(KibbleBit, source)
+ acquire(kibble_bit, source)
branch = find_branch(date, gpath)
if not branch:
@@ -178,7 +178,7 @@ def scan(KibbleBit, source):
branch_exists = check_branch(gpath, date, branch)
if not branch_exists:
- KibbleBit.pprint("Not trunk either (bad repo?), skipping")
+ kibble_bit.pprint("Not trunk either (bad repo?), skipping")
release(
source,
"Could not do evolutionary scan of code",
@@ -207,10 +207,10 @@ def scan(KibbleBit, source):
dhash = hashlib.sha224(
(source["sourceID"] + date).encode("ascii", "replace")
).hexdigest()
- found = KibbleBit.exists("evolution", dhash)
+ found = kibble_bit.exists("evolution", dhash)
if not found:
checkout(gpath, date, branch)
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Running cloc on %s (%s) at %s"
% (gname, source["sourceURL"], date)
)
@@ -229,7 +229,7 @@ def scan(KibbleBit, source):
"cost": cost,
"languages": languages,
}
- KibbleBit.index("evolution", dhash, js)
+ kibble_bit.index("evolution", dhash, js)
quarter -= 3
if quarter <= 0:
quarter += 12
@@ -238,9 +238,9 @@ def scan(KibbleBit, source):
# decrease month by 3
now = time.mktime(datetime.date(year, quarter, 1).timetuple())
except Exception as e:
- KibbleBit.pprint(e)
+ kibble_bit.pprint(e)
release(
- KibbleBit,
+ kibble_bit,
source,
"Evolution scan failed at "
+ time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
@@ -249,7 +249,7 @@ def scan(KibbleBit, source):
return
release(
- KibbleBit,
+ kibble_bit,
source,
"Evolution scan completed at "
+ time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
diff --git a/kibble/scanners/scanners/git-sloc.py b/kibble/scanners/scanners/git-sloc.py
index d963357..8b44c54 100644
--- a/kibble/scanners/scanners/git-sloc.py
+++ b/kibble/scanners/scanners/git-sloc.py
@@ -37,12 +37,12 @@ def accepts(source):
return False
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
rid = source["sourceID"]
url = source["sourceURL"]
rootpath = "%s/%s/git" % (
- KibbleBit.config["scanner"]["scratchdir"],
+ kibble_bit.config["scanner"]["scratchdir"],
source["organisation"],
)
gpath = os.path.join(rootpath, rid)
@@ -55,16 +55,16 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
try:
branch = git.defaultBranch(source, gpath)
subprocess.call("cd %s && git checkout %s" % (gpath, branch), shell=True)
- except:
- KibbleBit.pprint("SLoC counter failed to find main branch for %s!!" % url)
+ except: # pylint: disable=bare-except
+ kibble_bit.pprint("SLoC counter failed to find main branch for %s!!" % url)
return False
- KibbleBit.pprint("Running SLoC count for %s" % url)
+ kibble_bit.pprint("Running SLoC count for %s" % url)
languages, codecount, comment, blank, years, cost = sloc.count(gpath)
sloc_ = {
@@ -84,4 +84,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/git-sync.py b/kibble/scanners/scanners/git-sync.py
index e2e9422..c1c40f6 100644
--- a/kibble/scanners/scanners/git-sync.py
+++ b/kibble/scanners/scanners/git-sync.py
@@ -30,18 +30,18 @@ def accepts(source):
if source["type"] == "git":
return True
# There are cases where we have a github repo, but don't wanna analyze the code, just issues
- if source["type"] == "github" and source.get("issuesonly", False) == False:
+ if source["type"] == "github" and source.get("issuesonly", False) is False:
return True
return False
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Get some vars, construct a data path for the repo
path = source["sourceID"]
url = source["sourceURL"]
rootpath = "%s/%s/git" % (
- KibbleBit.config["scanner"]["scratchdir"],
+ kibble_bit.config["scanner"]["scratchdir"],
source["organisation"],
)
@@ -50,20 +50,20 @@ def scan(KibbleBit, source):
try:
os.makedirs(rootpath, exist_ok=True)
print("Created root path %s" % rootpath)
- except Exception as err:
+ except: # pylint: disable=bare-except
source["steps"]["sync"] = {
"time": time.time(),
"status": "Could not create root scratch dir - permision denied?",
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
# This is were the repo should be cloned
datapath = os.path.join(rootpath, path)
- KibbleBit.pprint("Checking out %s as %s" % (url, path))
+ kibble_bit.pprint("Checking out %s as %s" % (url, path))
try:
source["steps"]["sync"] = {
@@ -72,14 +72,14 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# If we already checked this out earlier, just sync it.
if os.path.exists(datapath):
- KibbleBit.pprint("Repo %s exists, fetching changes..." % datapath)
+ kibble_bit.pprint("Repo %s exists, fetching changes..." % datapath)
# Do we have a default branch here?
- branch = git.defaultBranch(source, datapath, KibbleBit)
+ branch = git.defaultBranch(source, datapath, kibble_bit)
if len(branch) == 0:
source["default_branch"] = branch
source["steps"]["sync"] = {
@@ -89,14 +89,14 @@ def scan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
- KibbleBit.pprint(
+ kibble_bit.update_source(source)
+ kibble_bit.pprint(
"No default branch found for %s (%s)"
% (source["sourceID"], source["sourceURL"])
)
return
- KibbleBit.pprint("Using branch %s" % branch)
+ kibble_bit.pprint("Using branch %s" % branch)
# Try twice checking out the main branch and fetching changes.
# Sometimes we need to clean up after older scanners, which is
# why we try twice. If first attempt fails, clean up and try again.
@@ -137,11 +137,11 @@ def scan(KibbleBit, source):
shell=True,
stderr=subprocess.STDOUT,
)
- except:
+ except: # pylint: disable=bare-except
pass
# This is a new repo, clone it!
else:
- KibbleBit.pprint("%s is new, cloning...!" % datapath)
+ kibble_bit.pprint("%s is new, cloning...!" % datapath)
subprocess.check_output(
"GIT_TERMINAL_PROMPT=0 cd %s && git clone %s %s"
% (rootpath, url, path),
@@ -150,8 +150,8 @@ def scan(KibbleBit, source):
)
except subprocess.CalledProcessError as err:
- KibbleBit.pprint("Repository sync failed (no master?)")
- KibbleBit.pprint(str(err.output))
+ kibble_bit.pprint("Repository sync failed (no master?)")
+ kibble_bit.pprint(str(err.output))
source["steps"]["sync"] = {
"time": time.time(),
"status": "Sync failed at "
@@ -160,7 +160,7 @@ def scan(KibbleBit, source):
"good": False,
"exception": str(err.output),
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
# All good, yay!
@@ -171,4 +171,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/github-issues.py b/kibble/scanners/scanners/github-issues.py
index 06bbde6..cdfcde1 100644
--- a/kibble/scanners/scanners/github-issues.py
+++ b/kibble/scanners/scanners/github-issues.py
@@ -124,29 +124,29 @@ def status_changed(stored_issue, issue):
return stored_issue["status"] != issue["status"]
-def update_issue(KibbleBit, issue):
- KibbleBit.append("issue", issue)
+def update_issue(kibble_bit, issue):
+ kibble_bit.append("issue", issue)
-def update_person(KibbleBit, person):
+def update_person(kibble_bit, person):
person["upsert"] = True
- KibbleBit.append("person", person)
+ kibble_bit.append("person", person)
-def scan(KibbleBit, source, firstAttempt=True):
+def scan(kibble_bit, source, first_attempt=True):
auth = None
people = {}
if "creds" in source:
- KibbleBit.pprint("Using auth for repo %s" % source["sourceURL"])
+ kibble_bit.pprint("Using auth for repo %s" % source["sourceURL"])
creds = source["creds"]
if creds and "username" in creds:
auth = (creds["username"], creds["password"])
TL = github.get_tokens_left(auth=auth)
- KibbleBit.pprint("Scanning for GitHub issues (%u tokens left on GitHub)" % TL)
+ kibble_bit.pprint("Scanning for GitHub issues (%u tokens left on GitHub)" % TL)
# Have we scanned before? If so, only do a 3 month scan here.
- doneBefore = False
+ done_before = False
if source.get("steps") and source["steps"].get("issues"):
- doneBefore = True
+ done_before = True
source["steps"]["issues"] = {
"time": time.time(),
"status": "Issue scan started at "
@@ -154,13 +154,13 @@ def scan(KibbleBit, source, firstAttempt=True):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
try:
- if doneBefore:
+ if done_before:
since = time.strftime(
"%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - (3 * 30 * 86400))
)
- KibbleBit.pprint("Fetching changes since %s" % since)
+ kibble_bit.pprint("Fetching changes since %s" % since)
issues = github.get_all(
source,
github.issues,
@@ -174,7 +174,7 @@ def scan(KibbleBit, source, firstAttempt=True):
params={"filter": "all", "state": "all"},
auth=auth,
)
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Fetched %s issues for %s" % (str(len(issues)), source["sourceURL"])
)
@@ -185,24 +185,24 @@ def scan(KibbleBit, source, firstAttempt=True):
source, issue, github.user(issue["user"]["url"], auth=auth)
)
people[issue["user"]["login"]] = person
- update_person(KibbleBit, person)
+ update_person(kibble_bit, person)
if "closed_by" in issue and not issue["closed_by"]["login"] in people:
closer = make_person(
source, issue, github.user(issue["closed_by"]["url"], auth=auth)
)
people[issue["closed_by"]["login"]] = closer
- update_person(KibbleBit, closer)
+ update_person(kibble_bit, closer)
doc = make_issue(source, issue, people)
dhash = doc["id"]
- if KibbleBit.exists("issue", dhash):
- es_doc = KibbleBit.get("issue", dhash)
+ if kibble_bit.exists("issue", dhash):
+ es_doc = kibble_bit.get("issue", dhash)
if not status_changed(es_doc, doc):
# KibbleBit.pprint("change %s seen already and status unchanged. Skipping." % issue['id'])
continue
- update_issue(KibbleBit, doc)
+ update_issue(kibble_bit, doc)
source["steps"]["issues"] = {
"time": time.time(),
@@ -211,28 +211,30 @@ def scan(KibbleBit, source, firstAttempt=True):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
except requests.HTTPError as e:
# If we errored out because of rate limiting, retry later, otherwise bail
- if firstAttempt:
+ if first_attempt:
sleeps = 0
if github.get_tokens_left(auth=auth) < 10:
- KibbleBit.pprint("Hit rate limits, trying to sleep it off!")
+ kibble_bit.pprint("Hit rate limits, trying to sleep it off!")
while github.get_tokens_left(auth=auth) < 10:
sleeps += 1
if sleeps > 24:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Slept for too long without finding a reset rate limit, giving up!"
)
break
time.sleep(300) # Sleep 5 min, then check again..
# If we have tokens, try one more time...
if github.get_tokens_left(auth=auth) > 10:
- scan(KibbleBit, source, False) # If this one fails, bail completely
+ scan(
+ kibble_bit, source, False
+ ) # If this one fails, bail completely
return
- KibbleBit.pprint("HTTP Error, rate limit exceeded?")
+ kibble_bit.pprint("HTTP Error, rate limit exceeded?")
source["steps"]["issues"] = {
"time": time.time(),
"status": "Issue scan failed at "
@@ -242,4 +244,4 @@ def scan(KibbleBit, source, firstAttempt=True):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/github-stats.py b/kibble/scanners/scanners/github-stats.py
index 504d151..ed4eda1 100644
--- a/kibble/scanners/scanners/github-stats.py
+++ b/kibble/scanners/scanners/github-stats.py
@@ -32,26 +32,26 @@ def accepts(source):
return False
-def getTime(string):
+def get_time(string):
""" Convert GitHub timestamp to epoch """
return time.mktime(
time.strptime(re.sub(r"Z", "", str(string)), "%Y-%m-%dT%H:%M:%S")
)
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Get some vars, construct a data path for the repo
url = source["sourceURL"]
auth = None
if "creds" in source:
- KibbleBit.pprint("Using auth for repo %s" % source["sourceURL"])
+ kibble_bit.pprint("Using auth for repo %s" % source["sourceURL"])
creds = source["creds"]
if creds and "username" in creds:
auth = (creds["username"], creds["password"])
else:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"GitHub stats requires auth, none provided. Ignoring this repo."
)
return
@@ -62,13 +62,13 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Get views
views = github.views(url, auth)
if "views" in views:
for el in views["views"]:
- ts = getTime(el["timestamp"])
+ ts = get_time(el["timestamp"])
shash = hashlib.sha224(
(
"%s-%s-%s-clones"
@@ -85,13 +85,13 @@ def scan(KibbleBit, source):
"ghtype": "views",
"id": shash,
}
- KibbleBit.append("ghstats", bit)
+ kibble_bit.append("ghstats", bit)
# Get clones
clones = github.clones(url, auth)
if "clones" in clones:
for el in clones["clones"]:
- ts = getTime(el["timestamp"])
+ ts = get_time(el["timestamp"])
shash = hashlib.sha224(
(
"%s-%s-%s-clones"
@@ -108,14 +108,14 @@ def scan(KibbleBit, source):
"ghtype": "clones",
"id": shash,
}
- KibbleBit.append("ghstats", bit)
+ kibble_bit.append("ghstats", bit)
# Get referrers
refs = github.referrers(url, auth)
if refs:
for el in refs:
el["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%S", time.time())
- ts = getTime(el["timestamp"])
+ ts = get_time(el["timestamp"])
shash = hashlib.sha224(
(
"%s-%s-%s-refs" % (source["organisation"], url, el["timestamp"])
@@ -131,7 +131,7 @@ def scan(KibbleBit, source):
"ghtype": "referrers",
"id": shash,
}
- KibbleBit.append("ghstats", bit)
- except:
+ kibble_bit.append("ghstats", bit)
+ except: # pylint: disable=bare-except
pass
# All done!
diff --git a/kibble/scanners/scanners/jenkins.py b/kibble/scanners/scanners/jenkins.py
index f7ec495..3db31ac 100644
--- a/kibble/scanners/scanners/jenkins.py
+++ b/kibble/scanners/scanners/jenkins.py
@@ -39,7 +39,7 @@ def accepts(source):
return False
-def scanJob(KibbleBit, source, job, creds):
+def scan_job(kibble_bit, source, job, creds):
""" Scans a single job for activity """
NOW = int(datetime.datetime.utcnow().timestamp())
jname = job["name"]
@@ -50,16 +50,14 @@ def scanJob(KibbleBit, source, job, creds):
"ascii", errors="replace"
)
).hexdigest()
- doc = None
- found = KibbleBit.exists("cijob", dhash)
# Get $jenkins/job/$job-name/json...
- jobURL = (
+ job_url = (
"%s/api/json?depth=2&tree=builds[number,status,timestamp,id,result,duration]"
% job["fullURL"]
)
- KibbleBit.pprint(jobURL)
- jobjson = jsonapi.get(jobURL, auth=creds)
+ kibble_bit.pprint(job_url)
+ jobjson = jsonapi.get(job_url, auth=creds)
# If valid JSON, ...
if jobjson:
@@ -72,15 +70,15 @@ def scanJob(KibbleBit, source, job, creds):
).hexdigest()
builddoc = None
try:
- builddoc = KibbleBit.get("ci_build", buildhash)
- except:
+ builddoc = kibble_bit.get("ci_build", buildhash)
+ except: # pylint: disable=bare-except
pass
# If this build already completed, no need to parse it again
if builddoc and builddoc.get("completed", False):
continue
- KibbleBit.pprint(
+ kibble_bit.pprint(
"[%s-%s] This is new or pending, analyzing..." % (jname, build["id"])
)
@@ -117,7 +115,7 @@ def scanJob(KibbleBit, source, job, creds):
"completed": completed,
"duration": build["duration"],
"job": jname,
- "jobURL": jobURL,
+ "job_url": job_url,
"status": status,
"started": int(build["timestamp"] / 1000),
"ci": "jenkins",
@@ -127,20 +125,20 @@ def scanJob(KibbleBit, source, job, creds):
"organisation": source["organisation"],
"upsert": True,
}
- KibbleBit.append("ci_build", doc)
+ kibble_bit.append("ci_build", doc)
# Yay, it worked!
return True
# Boo, it failed!
- KibbleBit.pprint("Fetching job data failed!")
+ kibble_bit.pprint("Fetching job data failed!")
return False
-class jenkinsThread(threading.Thread):
+class Jenkinsthread(threading.Thread):
""" Generic thread class for scheduling multiple scans at once """
def __init__(self, block, KibbleBit, source, creds, jobs):
- super(jenkinsThread, self).__init__()
+ super(Jenkinsthread, self).__init__()
self.block = block
self.KibbleBit = KibbleBit
self.creds = creds
@@ -148,8 +146,8 @@ class jenkinsThread(threading.Thread):
self.jobs = jobs
def run(self):
- badOnes = 0
- while len(self.jobs) > 0 and badOnes <= 50:
+ bad_ones = 0
+ while len(self.jobs) > 0 and bad_ones <= 50:
self.block.acquire()
try:
job = self.jobs.pop(0)
@@ -165,12 +163,12 @@ class jenkinsThread(threading.Thread):
ssource = dict(self.source)
if jfolder:
ssource["sourceURL"] += "/job/" + jfolder
- if not scanJob(self.KibbleBit, ssource, job, self.creds):
+ if not scan_job(self.KibbleBit, ssource, job, self.creds):
self.KibbleBit.pprint(
"[%s] This borked, trying another one" % job["name"]
)
- badOnes += 1
- if badOnes > 100:
+ bad_ones += 1
+ if bad_ones > 100:
self.KibbleBit.pprint("Too many errors, bailing!")
self.source["steps"]["issues"] = {
"time": time.time(),
@@ -179,13 +177,13 @@ class jenkinsThread(threading.Thread):
"running": False,
"good": False,
}
- self.KibbleBit.updateSource(self.source)
+ self.KibbleBit.update_source(self.source)
return
else:
- badOnes = 0
+ bad_ones = 0
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Simple URL check
jenkins = re.match(r"(https?://.+)", source["sourceURL"])
if jenkins:
@@ -196,17 +194,16 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
- pendingJobs = []
- KibbleBit.pprint("Parsing Jenkins activity at %s" % source["sourceURL"])
+ kibble_bit.pprint("Parsing Jenkins activity at %s" % source["sourceURL"])
source["steps"]["issues"] = {
"time": time.time(),
"status": "Downloading changeset",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Jenkins may neeed credentials
creds = None
@@ -219,15 +216,15 @@ def scan(KibbleBit, source):
creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"])
# Get the job list
- sURL = source["sourceURL"]
- KibbleBit.pprint("Getting job list...")
+ s_url = source["sourceURL"]
+ kibble_bit.pprint("Getting job list...")
jobsjs = jsonapi.get(
- "%s/api/json?tree=jobs[name,color]&depth=1" % sURL, auth=creds
+ "%s/api/json?tree=jobs[name,color]&depth=1" % s_url, auth=creds
)
# Get the current queue
- KibbleBit.pprint("Getting job queue...")
- queuejs = jsonapi.get("%s/queue/api/json?depth=1" % sURL, auth=creds)
+ kibble_bit.pprint("Getting job queue...")
+ queuejs = jsonapi.get("%s/queue/api/json?depth=1" % s_url, auth=creds)
# Save queue snapshot
NOW = int(datetime.datetime.utcnow().timestamp())
@@ -256,7 +253,7 @@ def scan(KibbleBit, source):
# Count how many jobs are building, find any folders...
actual_jobs, building = get_all_jobs(
- KibbleBit, source, jobsjs.get("jobs", []), creds
+ kibble_bit, source, jobsjs.get("jobs", []), creds
)
# Write up a queue doc
@@ -275,16 +272,16 @@ def scan(KibbleBit, source):
"organisation": source["organisation"],
"upsert": True,
}
- KibbleBit.append("ci_queue", queuedoc)
+ kibble_bit.append("ci_queue", queuedoc)
- pendingJobs = actual_jobs
- KibbleBit.pprint("Found %u jobs in Jenkins" % len(pendingJobs))
+ pending_jobs = actual_jobs
+ kibble_bit.pprint("Found %u jobs in Jenkins" % len(pending_jobs))
threads = []
block = threading.Lock()
- KibbleBit.pprint("Scanning jobs using 4 sub-threads")
+ kibble_bit.pprint("Scanning jobs using 4 sub-threads")
for i in range(0, 4):
- t = jenkinsThread(block, KibbleBit, source, creds, pendingJobs)
+ t = Jenkinsthread(block, kibble_bit, source, creds, pending_jobs)
threads.append(t)
t.start()
@@ -292,7 +289,7 @@ def scan(KibbleBit, source):
t.join()
# We're all done, yaay
- KibbleBit.pprint("Done scanning %s" % source["sourceURL"])
+ kibble_bit.pprint("Done scanning %s" % source["sourceURL"])
source["steps"]["issues"] = {
"time": time.time(),
@@ -301,10 +298,10 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
-def get_all_jobs(KibbleBit, source, joblist, creds):
+def get_all_jobs(kibble_bit, source, joblist, creds):
real_jobs = []
building = 0
for job in joblist:
@@ -314,30 +311,30 @@ def get_all_jobs(KibbleBit, source, joblist, creds):
"jenkins.branch.OrganizationFolder",
"org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject",
]:
- KibbleBit.pprint("%s is a jobs folder, expanding..." % job["name"])
- csURL = "%s/job/%s" % (
+ kibble_bit.pprint("%s is a jobs folder, expanding..." % job["name"])
+ cs_url = "%s/job/%s" % (
source["sourceURL"],
urllib.parse.quote(job["name"].replace("/", "%2F")),
)
try:
child_jobs = jsonapi.get(
- "%s/api/json?tree=jobs[name,color]&depth=1" % csURL, auth=creds
+ "%s/api/json?tree=jobs[name,color]&depth=1" % cs_url, auth=creds
)
csource = dict(source)
- csource["sourceURL"] = csURL
+ csource["sourceURL"] = cs_url
if not csource.get("folder"):
csource["folder"] = job["name"]
else:
csource["folder"] += "-" + job["name"]
cjobs, cbuilding = get_all_jobs(
- KibbleBit, csource, child_jobs.get("jobs", []), creds
+ kibble_bit, csource, child_jobs.get("jobs", []), creds
)
building += cbuilding
for cjob in cjobs:
real_jobs.append(cjob)
- except:
- KibbleBit.pprint("Couldn't get child jobs, bailing")
- print("%s/api/json?tree=jobs[name,color]&depth=1" % csURL)
+ except: # pylint: disable=bare-except
+ kibble_bit.pprint("Couldn't get child jobs, bailing")
+ print("%s/api/json?tree=jobs[name,color]&depth=1" % cs_url)
# Or standard job?
else:
# Is it building?
diff --git a/kibble/scanners/scanners/jira.py b/kibble/scanners/scanners/jira.py
index 41dafa6..14789c9 100644
--- a/kibble/scanners/scanners/jira.py
+++ b/kibble/scanners/scanners/jira.py
@@ -43,7 +43,7 @@ def accepts(source):
return False
-def getTime(string):
+def get_time(string):
return time.mktime(
time.strptime(re.sub(r"\..*", "", str(string)), "%Y-%m-%dT%H:%M:%S")
)
@@ -121,7 +121,7 @@ def pchange(js):
return False
-def scanTicket(KibbleBit, key, u, source, creds, openTickets):
+def scan_ticket(kibble_bit, key, u, source, creds, open_tickets):
""" Scans a single ticket for activity and people """
dhash = hashlib.sha224(
@@ -129,8 +129,7 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
"ascii", errors="replace"
)
).hexdigest()
- found = True
- parseIt = False
+ parse_it = False
# the 'domain' var we try to figure out here is used
# for faking email addresses and keep them unique,
@@ -140,62 +139,62 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
if m:
domain = m.group(1)
- found = KibbleBit.exists("issue", dhash)
+ found = kibble_bit.exists("issue", dhash)
if not found:
- KibbleBit.pprint("[%s] We've never seen this ticket before, parsing..." % key)
- parseIt = True
+ kibble_bit.pprint("[%s] We've never seen this ticket before, parsing..." % key)
+ parse_it = True
else:
- ticket = KibbleBit.get("issue", dhash)
- if ticket["status"] == "closed" and key in openTickets:
- KibbleBit.pprint("[%s] Ticket was reopened, reparsing" % key)
- parseIt = True
- elif ticket["status"] == "open" and not key in openTickets:
- KibbleBit.pprint("[%s] Ticket was recently closed, parsing it" % key)
- parseIt = True
+ ticket = kibble_bit.get("issue", dhash)
+ if ticket["status"] == "closed" and key in open_tickets:
+ kibble_bit.pprint("[%s] Ticket was reopened, reparsing" % key)
+ parse_it = True
+ elif ticket["status"] == "open" and not key in open_tickets:
+ kibble_bit.pprint("[%s] Ticket was recently closed, parsing it" % key)
+ parse_it = True
else:
if (
ticket["issueCreator"] == "unknown@kibble"
or ticket["issueCloser"] == "unknown@kibble"
): # Gotta redo these!
- parseIt = True
- KibbleBit.pprint(
+ parse_it = True
+ kibble_bit.pprint(
"[%s] Ticket contains erroneous data from a previous scan, reparsing"
% key
)
# This is just noise!
# KibbleBit.pprint("[%s] Ticket hasn't changed, ignoring..." % key)
- if parseIt:
- KibbleBit.pprint("[%s] Parsing data from JIRA at %s..." % (key, domain))
- queryURL = (
+ if parse_it:
+ kibble_bit.pprint("[%s] Parsing data from JIRA at %s..." % (key, domain))
+ query_url = (
"%s/rest/api/2/issue/%s?fields=creator,reporter,status,issuetype,summary,assignee,resolutiondate,created,priority,changelog,comment,resolution,votes&expand=changelog"
% (u, key)
)
- jiraURL = "%s/browse/%s" % (u, key)
+ jira_url = "%s/browse/%s" % (u, key)
try:
- tjson = jsonapi.get(queryURL, auth=creds)
+ tjson = jsonapi.get(query_url, auth=creds)
if not tjson:
- KibbleBit.pprint("%s does not exist (404'ed)" % key)
+ kibble_bit.pprint("%s does not exist (404'ed)" % key)
return False
except requests.exceptions.ConnectionError as err:
- KibbleBit.pprint(f"Connection error: {err}, skipping this ticket for now!")
+ kibble_bit.pprint(f"Connection error: {err}, skipping this ticket for now!")
return False
st, closer = wasclosed(tjson)
if st and not closer:
- KibbleBit.pprint("Closed but no closer??")
- closerEmail = None
+ kibble_bit.pprint("Closed but no closer??")
+ closer_email = None
status = "closed" if st else "open"
# Make sure we actually have field data to work with
if not tjson.get("fields") or not tjson["fields"].get("created"):
- KibbleBit.pprint(
+ kibble_bit.pprint(
"[%s] JIRA response is missing field data, ignoring ticket." % key
)
return False
- cd = getTime(tjson["fields"]["created"])
+ cd = get_time(tjson["fields"]["created"])
rd = (
- getTime(tjson["fields"]["resolutiondate"])
+ get_time(tjson["fields"]["resolutiondate"])
if "resolutiondate" in tjson["fields"] and tjson["fields"]["resolutiondate"]
else None
)
@@ -221,40 +220,40 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
title = tjson["fields"]["summary"]
if closer:
# print("Parsing closer")
- closerEmail = (
+ closer_email = (
closer.get("emailAddress", closer.get("name"))
.replace(" dot ", ".", 10)
.replace(" at ", "@", 1)
)
- if not "@" in closerEmail:
- closerEmail = "%s@%s" % (closerEmail, domain)
- displayName = closer.get("displayName", "Unkown")
- if displayName and len(displayName) > 0:
+ if not "@" in closer_email:
+ closer_email = "%s@%s" % (closer_email, domain)
+ display_name = closer.get("displayName", "Unkown")
+ if display_name and len(display_name) > 0:
# Add to people db
pid = hashlib.sha1(
- ("%s%s" % (source["organisation"], closerEmail)).encode(
+ ("%s%s" % (source["organisation"], closer_email)).encode(
"ascii", errors="replace"
)
).hexdigest()
jsp = {
- "name": displayName,
- "email": closerEmail,
+ "name": display_name,
+ "email": closer_email,
"organisation": source["organisation"],
"id": pid,
"upsert": True,
}
- KibbleBit.append("person", jsp)
+ kibble_bit.append("person", jsp)
if creator:
creator = creator.replace(" dot ", ".", 10).replace(" at ", "@", 1)
if not "@" in creator:
creator = "%s@%s" % (creator, domain)
- displayName = (
+ display_name = (
tjson["fields"]["reporter"]["displayName"]
if tjson["fields"]["reporter"]
else None
)
- if displayName and len(displayName) > 0:
+ if display_name and len(display_name) > 0:
# Add to people db
pid = hashlib.sha1(
("%s%s" % (source["organisation"], creator)).encode(
@@ -262,13 +261,13 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
)
).hexdigest()
jsp = {
- "name": displayName,
+ "name": display_name,
"email": creator,
"organisation": source["organisation"],
"id": pid,
"upsert": True,
}
- KibbleBit.append("person", jsp)
+ kibble_bit.append("person", jsp)
if assignee and not "@" in assignee:
assignee = "%s@%s" % (assignee, domain)
jso = {
@@ -276,12 +275,12 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
"key": key,
"organisation": source["organisation"],
"sourceID": source["sourceID"],
- "url": jiraURL,
+ "url": jira_url,
"status": status,
"created": cd,
"closed": rd,
"issuetype": "issue",
- "issueCloser": closerEmail,
+ "issueCloser": closer_email,
"createdDate": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(cd)),
"closedDate": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(rd))
if rd
@@ -294,7 +293,7 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
"comments": comments,
"title": title,
}
- KibbleBit.append("issue", jso)
+ kibble_bit.append("issue", jso)
return True
@@ -304,19 +303,19 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets):
# return False
-class jiraThread(threading.Thread):
- def __init__(self, block, KibbleBit, source, creds, pt, ot):
- super(jiraThread, self).__init__()
+class JiraThread(threading.Thread):
+ def __init__(self, block, kibble_bit, source, creds, pt, ot):
+ super(JiraThread, self).__init__()
self.block = block
- self.KibbleBit = KibbleBit
+ self.KibbleBit = kibble_bit
self.creds = creds
self.source = source
self.pendingTickets = pt
self.openTickets = ot
def run(self):
- badOnes = 0
- while len(self.pendingTickets) > 0 and badOnes <= 50:
+ bad_ones = 0
+ while len(self.pendingTickets) > 0 and bad_ones <= 50:
# print("%u elements left to count" % len(pendingTickets))
self.block.acquire()
try:
@@ -329,12 +328,12 @@ class jiraThread(threading.Thread):
self.block.release()
return
self.block.release()
- if not scanTicket(
+ if not scan_ticket(
self.KibbleBit, rl[0], rl[1], rl[2], self.creds, self.openTickets
):
self.KibbleBit.pprint("[%s] This borked, trying another one" % rl[0])
- badOnes += 1
- if badOnes > 100:
+ bad_ones += 1
+ if bad_ones > 100:
self.KibbleBit.pprint("Too many errors, bailing!")
self.source["steps"]["issues"] = {
"time": time.time(),
@@ -343,13 +342,13 @@ class jiraThread(threading.Thread):
"running": False,
"good": False,
}
- self.KibbleBit.updateSource(self.source)
+ self.KibbleBit.update_source(self.source)
return
else:
- badOnes = 0
+ bad_ones = 0
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
jira = re.match(r"(https?://.+)/browse/([A-Z0-9]+)", source["sourceURL"])
if jira:
@@ -363,7 +362,7 @@ def scan(KibbleBit, source):
):
creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"])
if not creds:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"JIRA at %s requires authentication, but none was found! Bailing."
% source["sourceURL"]
)
@@ -373,7 +372,7 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
source["steps"]["issues"] = {
@@ -382,74 +381,75 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
- pendingTickets = []
- KibbleBit.pprint("Parsing JIRA activity at %s" % source["sourceURL"])
+ pending_tickets = []
+ kibble_bit.pprint("Parsing JIRA activity at %s" % source["sourceURL"])
source["steps"]["issues"] = {
"time": time.time(),
"status": "Downloading changeset",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Get base URL, list and domain to parse
u = jira.group(1)
instance = jira.group(2)
- lastTicket = 0
- latestURL = (
+ last_ticket = 0
+ latest_url = (
"%s/rest/api/2/search?jql=project=%s+order+by+createdDate+DESC&fields=id,key&maxResults=1"
% (u, instance)
)
js = None
- js = jsonapi.get(latestURL, auth=creds)
+ js = jsonapi.get(latest_url, auth=creds)
if "issues" in js and len(js["issues"]) == 1:
key = js["issues"][0]["key"]
m = re.search(r"-(\d+)$", key)
if m:
- lastTicket = int(m.group(1))
+ last_ticket = int(m.group(1))
- openTickets = []
- startAt = 0
- badTries = 0
- while True and badTries < 10:
- openURL = (
+ open_tickets = []
+ start_at = 0
+ bad_tries = 0
+ while bad_tries < 10:
+ open_url = (
"%s/rest/api/2/search?jql=project=%s+and+status=open+order+by+createdDate+ASC&fields=id,key&maxResults=100&startAt=%u"
- % (u, instance, startAt)
+ % (u, instance, start_at)
)
# print(openURL)
try:
- ojs = jsonapi.get(openURL, auth=creds)
+ ojs = jsonapi.get(open_url, auth=creds)
if not "issues" in ojs or len(ojs["issues"]) == 0:
break
for item in ojs["issues"]:
- openTickets.append(item["key"])
- KibbleBit.pprint("Found %u open tickets" % len(openTickets))
- startAt += 100
- except:
- KibbleBit.pprint("JIRA borked, retrying")
- badTries += 1
- KibbleBit.pprint("Found %u open tickets" % len(openTickets))
-
- badOnes = 0
- for i in reversed(range(1, lastTicket + 1)):
+ open_tickets.append(item["key"])
+ kibble_bit.pprint("Found %u open tickets" % len(open_tickets))
+ start_at += 100
+ except: # pylint: disable=bare-except
+ kibble_bit.pprint("JIRA borked, retrying")
+ bad_tries += 1
+ kibble_bit.pprint("Found %u open tickets" % len(open_tickets))
+
+ for i in reversed(range(1, last_ticket + 1)):
key = "%s-%u" % (instance, i)
- pendingTickets.append([key, u, source])
+ pending_tickets.append([key, u, source])
threads = []
block = threading.Lock()
- KibbleBit.pprint("Scanning tickets using 4 sub-threads")
+ kibble_bit.pprint("Scanning tickets using 4 sub-threads")
for i in range(0, 4):
- t = jiraThread(block, KibbleBit, source, creds, pendingTickets, openTickets)
+ t = JiraThread(
+ block, kibble_bit, source, creds, pending_tickets, open_tickets
+ )
threads.append(t)
t.start()
for t in threads:
t.join()
- KibbleBit.pprint("Done scanning %s" % source["sourceURL"])
+ kibble_bit.pprint("Done scanning %s" % source["sourceURL"])
source["steps"]["issues"] = {
"time": time.time(),
@@ -458,4 +458,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/pipermail.py b/kibble/scanners/scanners/pipermail.py
index 606a97e..9a0c598 100644
--- a/kibble/scanners/scanners/pipermail.py
+++ b/kibble/scanners/scanners/pipermail.py
@@ -43,11 +43,11 @@ def accepts(source):
return False
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
url = source["sourceURL"]
pipermail = re.match(r"(https?://.+/(archives|pipermail)/.+?)/?$", url)
if pipermail:
- KibbleBit.pprint("Scanning Pipermail source %s" % url)
+ kibble_bit.pprint("Scanning Pipermail source %s" % url)
skipped = 0
source["steps"]["mail"] = {
@@ -56,10 +56,10 @@ def scan(KibbleBit, source):
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
dt = time.gmtime(time.time())
- firstYear = 1970
+ first_year = 1970
year = dt[0]
month = dt[1]
if month <= 0:
@@ -70,7 +70,7 @@ def scan(KibbleBit, source):
knowns = {}
# While we have older archives, continue to parse
- monthNames = [
+ month_names = [
"December",
"January",
"February",
@@ -85,16 +85,15 @@ def scan(KibbleBit, source):
"November",
"December",
]
- while firstYear <= year:
- gzurl = "%s/%04u-%s.txt.gz" % (url, year, monthNames[month])
+ while first_year <= year:
+ gzurl = "%s/%04u-%s.txt.gz" % (url, year, month_names[month])
pd = datetime.date(year, month, 1).timetuple()
dhash = hashlib.sha224(
("%s %s" % (source["organisation"], gzurl)).encode(
"ascii", errors="replace"
)
).hexdigest()
- found = False
- found = KibbleBit.exists("mailstats", dhash)
+ found = kibble_bit.exists("mailstats", dhash)
if (
months <= 1 or not found
): # Always parse this month's stats and the previous month :)
@@ -153,7 +152,7 @@ def scan(KibbleBit, source):
if m:
name = m.group(1).replace('"', "").strip()
sender = m.group(2)
- if not sender in posters:
+ if sender not in posters:
posters[sender] = {"name": name, "email": sender}
senders[message.get("message-id", "??")] = sender
mdate = email.utils.parsedate_tz(message["date"])
@@ -161,15 +160,15 @@ def scan(KibbleBit, source):
"%Y/%m/%d %H:%M:%S",
time.gmtime(email.utils.mktime_tz(mdate)),
)
- if not sender in knowns:
+ if sender not in knowns:
sid = hashlib.sha1(
("%s%s" % (source["organisation"], sender)).encode(
"ascii", errors="replace"
)
).hexdigest()
- knowns[sender] = KibbleBit.exists("person", sid)
- if not sender in knowns:
- KibbleBit.append(
+ knowns[sender] = kibble_bit.exists("person", sid)
+ if sender not in knowns:
+ kibble_bit.append(
"person",
{
"name": name,
@@ -196,20 +195,17 @@ def scan(KibbleBit, source):
"ts": email.utils.mktime_tz(mdate),
"id": message["message-id"],
}
- KibbleBit.append("email", jse)
+ kibble_bit.append("email", jse)
- for sender in posters:
- no_posters += 1
+ no_posters = len(posters)
+ topics = len(rawtopics)
i = 0
- topics = 0
- for key in rawtopics:
- topics += 1
for key in reversed(sorted(rawtopics, key=lambda x: x)):
val = rawtopics[key]
i += 1
if i > 10:
break
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Found top 10: %s (%s emails)" % (key, val)
)
shash = hashlib.sha224(
@@ -238,7 +234,7 @@ def scan(KibbleBit, source):
"ts": time.mktime(pd),
"id": mlhash,
}
- KibbleBit.index("mailtop", mlhash, jst)
+ kibble_bit.index("mailtop", mlhash, jst)
jso = {
"organisation": source["organisation"],
@@ -249,24 +245,24 @@ def scan(KibbleBit, source):
"emails": emails,
"topics": topics,
}
- KibbleBit.index("mailstats", dhash, jso)
+ kibble_bit.index("mailstats", dhash, jso)
os.unlink(mailFile)
except Exception as err:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Couldn't parse %s, skipping: %s" % (gzurl, err)
)
skipped += 1
if skipped > 12:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"12 skips in a row, breaking off (no more data?)"
)
break
else:
- KibbleBit.pprint("Couldn't find %s, skipping." % gzurl)
+ kibble_bit.pprint("Couldn't find %s, skipping." % gzurl)
skipped += 1
if skipped > 12:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"12 skips in a row, breaking off (no more data?)"
)
break
@@ -282,13 +278,13 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
else:
- KibbleBit.pprint("Invalid Pipermail URL detected: %s" % url, True)
+ kibble_bit.pprint("Invalid Pipermail URL detected: %s" % url, True)
source["steps"]["mail"] = {
"time": time.time(),
"status": "Invalid or malformed URL detected!",
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/ponymail-kpe.py b/kibble/scanners/scanners/ponymail-kpe.py
index 50edac7..8db47c2 100644
--- a/kibble/scanners/scanners/ponymail-kpe.py
+++ b/kibble/scanners/scanners/ponymail-kpe.py
@@ -47,11 +47,11 @@ def accepts(source):
return False
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Validate URL first
url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source["sourceURL"])
if not url:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Malformed or invalid Pony Mail URL passed to scanner: %s"
% source["sourceURL"]
)
@@ -61,11 +61,11 @@ def scan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
- if not "azure" in KibbleBit.config and not "picoapi" in KibbleBit.config:
- KibbleBit.pprint(
+ if not "azure" in kibble_bit.config and not "picoapi" in kibble_bit.config:
+ kibble_bit.pprint(
"No Azure/picoAPI creds configured, skipping key phrase extraction"
)
return
@@ -74,15 +74,15 @@ def scan(KibbleBit, source):
if "creds" in source and source["creds"]:
cookie = source["creds"].get("cookie", None)
- rootURL = re.sub(r"list.html.+", "", source["sourceURL"])
+ root_url = re.sub(r"list.html.+", "", source["sourceURL"])
query = {
"query": {"bool": {"must": [{"term": {"sourceID": source["sourceID"]}}]}},
"sort": [{"ts": "desc"}],
}
# Get an initial count of commits
- res = KibbleBit.broker.DB.search(
- index=KibbleBit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4
+ res = kibble_bit.broker.DB.search(
+ index=kibble_bit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4
)
ec = 0
hits = []
@@ -93,30 +93,31 @@ def scan(KibbleBit, source):
if ec > MAX_COUNT:
break
if "kpe" not in eml:
- emlurl = "%s/api/email.lua?id=%s" % (rootURL, eml["id"])
- KibbleBit.pprint("Fetching %s" % emlurl)
+ emlurl = "%s/api/email.lua?id=%s" % (root_url, eml["id"])
+ kibble_bit.pprint("Fetching %s" % emlurl)
rv = None
try:
rv = jsonapi.get(emlurl, cookie=cookie)
if rv and "body" in rv:
hits.append([hit["_id"], rv["body"], eml])
except Exception as err:
- KibbleBit.pprint(f"Server error: {err}, skipping this email")
+ kibble_bit.pprint(f"Server error: {err}, skipping this email")
bodies = []
for hit in hits:
body = hit[1]
- bid = hit[0]
+ # bid = hit[0]
bodies.append(body)
if bodies:
- if "watson" in KibbleBit.config:
+ KPEs = None
+ if "watson" in kibble_bit.config:
pass # Haven't written this yet
- elif "azure" in KibbleBit.config:
- KPEs = kpe.azureKPE(KibbleBit, bodies)
- elif "picoapi" in KibbleBit.config:
- KPEs = kpe.picoKPE(KibbleBit, bodies)
+ elif "azure" in kibble_bit.config:
+ KPEs = kpe.azureKPE(kibble_bit, bodies)
+ elif "picoapi" in kibble_bit.config:
+ KPEs = kpe.picoKPE(kibble_bit, bodies)
if not KPEs:
- KibbleBit.pprint("Hit rate limit, not trying further emails for now.")
+ kibble_bit.pprint("Hit rate limit, not trying further emails for now.")
a = 0
for hit in hits:
@@ -128,7 +129,7 @@ def scan(KibbleBit, source):
kpe_ = ["_NULL_"]
eml["kpe"] = kpe_
print("Key phrases for %s: %s" % (bid, ", ".join(kpe_)))
- KibbleBit.index("email", bid, eml)
+ kibble_bit.index("email", bid, eml)
else:
- KibbleBit.pprint("No emails to analyze")
- KibbleBit.pprint("Done with key phrase extraction")
+ kibble_bit.pprint("No emails to analyze")
+ kibble_bit.pprint("Done with key phrase extraction")
diff --git a/kibble/scanners/scanners/ponymail-tone.py b/kibble/scanners/scanners/ponymail-tone.py
index e0dea32..4ae9330 100644
--- a/kibble/scanners/scanners/ponymail-tone.py
+++ b/kibble/scanners/scanners/ponymail-tone.py
@@ -44,11 +44,11 @@ def accepts(source):
return False
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Validate URL first
url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source["sourceURL"])
if not url:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Malformed or invalid Pony Mail URL passed to scanner: %s"
% source["sourceURL"]
)
@@ -58,15 +58,15 @@ def scan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
if (
- not "watson" in KibbleBit.config
- and not "azure" in KibbleBit.config
- and not "picoapi" in KibbleBit.config
+ not "watson" in kibble_bit.config
+ and not "azure" in kibble_bit.config
+ and not "picoapi" in kibble_bit.config
):
- KibbleBit.pprint(
+ kibble_bit.pprint(
"No Watson/Azure/picoAPI creds configured, skipping tone analyzer"
)
return
@@ -75,15 +75,15 @@ def scan(KibbleBit, source):
if "creds" in source and source["creds"]:
cookie = source["creds"].get("cookie", None)
- rootURL = re.sub(r"list.html.+", "", source["sourceURL"])
+ root_url = re.sub(r"list.html.+", "", source["sourceURL"])
query = {
"query": {"bool": {"must": [{"term": {"sourceID": source["sourceID"]}}]}},
"sort": [{"ts": "desc"}],
}
# Get an initial count of commits
- res = KibbleBit.broker.DB.search(
- index=KibbleBit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4
+ res = kibble_bit.broker.DB.search(
+ index=kibble_bit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4
)
ec = 0
hits = []
@@ -94,30 +94,30 @@ def scan(KibbleBit, source):
if ec > MAX_COUNT:
break
if "mood" not in eml:
- emlurl = "%s/api/email.lua?id=%s" % (rootURL, eml["id"])
- KibbleBit.pprint("Fetching %s" % emlurl)
- rv = None
+ emlurl = "%s/api/email.lua?id=%s" % (root_url, eml["id"])
+ kibble_bit.pprint("Fetching %s" % emlurl)
try:
rv = jsonapi.get(emlurl, cookie=cookie)
if rv and "body" in rv:
hits.append([hit["_id"], rv["body"], eml])
except Exception as err:
- KibbleBit.pprint(f"Server error: {err}, skipping this email")
+ kibble_bit.pprint(f"Server error: {err}, skipping this email")
bodies = []
for hit in hits:
body = hit[1]
- bid = hit[0]
+ # bid = hit[0]
bodies.append(body)
if bodies:
- if "watson" in KibbleBit.config:
- moods = tone.watsonTone(KibbleBit, bodies)
- elif "azure" in KibbleBit.config:
- moods = tone.azureTone(KibbleBit, bodies)
- elif "picoapi" in KibbleBit.config:
- moods = tone.picoTone(KibbleBit, bodies)
+ moods = None
+ if "watson" in kibble_bit.config:
+ moods = tone.watsonTone(kibble_bit, bodies)
+ elif "azure" in kibble_bit.config:
+ moods = tone.azureTone(kibble_bit, bodies)
+ elif "picoapi" in kibble_bit.config:
+ moods = tone.picoTone(kibble_bit, bodies)
if not moods:
- KibbleBit.pprint("Hit rate limit, not trying further emails for now.")
+ kibble_bit.pprint("Hit rate limit, not trying further emails for now.")
a = 0
for hit in hits:
@@ -131,7 +131,7 @@ def scan(KibbleBit, source):
if s > hm[0]:
hm = [s, m]
print("Likeliest overall mood for %s: %s" % (bid, hm[1]))
- KibbleBit.index("email", bid, eml)
+ kibble_bit.index("email", bid, eml)
else:
- KibbleBit.pprint("No emails to analyze")
- KibbleBit.pprint("Done with tone analysis")
+ kibble_bit.pprint("No emails to analyze")
+ kibble_bit.pprint("Done with tone analysis")
diff --git a/kibble/scanners/scanners/ponymail.py b/kibble/scanners/scanners/ponymail.py
index 92a5b0b..69a49ca 100644
--- a/kibble/scanners/scanners/ponymail.py
+++ b/kibble/scanners/scanners/ponymail.py
@@ -45,42 +45,42 @@ def accepts(source):
return False
-def countSubs(struct, kids=0):
+def count_subs(struct, kids=0):
""" Counts replies in a thread """
if "children" in struct and len(struct["children"]) > 0:
for child in struct["children"]:
kids += 1
- kids += countSubs(child)
+ kids += count_subs(child)
return kids
-def repliedTo(emails, struct):
- myList = {}
+def replied_to(emails, struct):
+ my_list = {}
for eml in struct:
- myID = eml["tid"]
+ my_id = eml["tid"]
if "children" in eml:
for child in eml["children"]:
- myList[child["tid"]] = myID
+ my_list[child["tid"]] = my_id
if len(child["children"]) > 0:
- cList = repliedTo(emails, child["children"])
- myList.update(cList)
- return myList
+ c_list = replied_to(emails, child["children"])
+ my_list.update(c_list)
+ return my_list
-def getSender(email):
+def get_sender(email):
sender = email["from"]
m = re.match(r"(.+)\s*<(.+)>", email["from"], flags=re.UNICODE)
if m:
- name = m.group(1).replace('"', "").strip()
+ # name = m.group(1).replace('"', "").strip()
sender = m.group(2)
return sender
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
# Validate URL first
url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source["sourceURL"])
if not url:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Malformed or invalid Pony Mail URL passed to scanner: %s"
% source["sourceURL"]
)
@@ -90,7 +90,7 @@ def scan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
# Pony Mail requires a UI cookie in order to work. Maked sure we have one!
@@ -98,7 +98,7 @@ def scan(KibbleBit, source):
if "creds" in source and source["creds"]:
cookie = source["creds"].get("cookie", None)
if not cookie:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Pony Mail instance at %s requires an authorized cookie, none found! Bailing."
% source["sourceURL"]
)
@@ -108,18 +108,18 @@ def scan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
# Notify scanner and DB that this is valid and we've begun parsing
- KibbleBit.pprint("%s is a valid Pony Mail address, parsing" % source["sourceURL"])
+ kibble_bit.pprint("%s is a valid Pony Mail address, parsing" % source["sourceURL"])
source["steps"]["mail"] = {
"time": time.time(),
"status": "Downloading Pony Mail statistics",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Get base URL, list and domain to parse
u = url.group(1)
@@ -128,7 +128,7 @@ def scan(KibbleBit, source):
# Get this month
dt = time.gmtime(time.time())
- firstYear = 1970
+ first_year = 1970
year = dt[0]
month = dt[1]
if month <= 0:
@@ -140,7 +140,7 @@ def scan(KibbleBit, source):
knowns = {}
# While we have older archives, continue to parse
- while firstYear <= year:
+ while first_year <= year:
statsurl = "%s/api/stats.lua?list=%s&domain=%s&d=%s" % (
u,
l,
@@ -153,36 +153,36 @@ def scan(KibbleBit, source):
)
).hexdigest()
found = False
- if KibbleBit.exists("mailstats", dhash):
+ if kibble_bit.exists("mailstats", dhash):
found = True
if months <= 1 or not found: # Always parse this month's stats :)
months += 1
- KibbleBit.pprint("Parsing %04u-%02u" % (year, month))
- KibbleBit.pprint(statsurl)
+ kibble_bit.pprint("Parsing %04u-%02u" % (year, month))
+ kibble_bit.pprint(statsurl)
pd = datetime.date(year, month, 1).timetuple()
try:
js = jsonapi.get(statsurl, cookie=cookie)
except Exception as err:
- KibbleBit.pprint(f"Server error: {err}, skipping this month")
+ kibble_bit.pprint(f"Server error: {err}, skipping this month")
month -= 1
if month <= 0:
month += 12
year -= 1
continue
if "firstYear" in js:
- firstYear = js["firstYear"]
+ first_year = js["firstYear"]
# print("First Year is %u" % firstYear)
else:
- KibbleBit.pprint("JSON was missing fields, aborting!")
+ kibble_bit.pprint("JSON was missing fields, aborting!")
break
- replyList = repliedTo(js["emails"], js["thread_struct"])
+ reply_list = replied_to(js["emails"], js["thread_struct"])
topics = js["no_threads"]
posters = {}
no_posters = 0
emails = len(js["emails"])
top10 = []
for eml in js["thread_struct"]:
- count = countSubs(eml, 0)
+ count = count_subs(eml, 0)
subject = ""
for reml in js["emails"]:
if reml["id"] == eml["tid"]:
@@ -202,7 +202,7 @@ def scan(KibbleBit, source):
i += 1
if i > 10:
break
- KibbleBit.pprint("Found top 10: %s (%s emails)" % (top[1], top[2]))
+ kibble_bit.pprint("Found top 10: %s (%s emails)" % (top[1], top[2]))
md = time.strftime("%Y/%m/%d %H:%M:%S", pd)
mlhash = hashlib.sha224(
(
@@ -221,7 +221,7 @@ def scan(KibbleBit, source):
"ts": time.mktime(pd),
"id": mlhash,
}
- KibbleBit.index("mailtop", mlhash, jst)
+ kibble_bit.index("mailtop", mlhash, jst)
for email in js["emails"]:
sender = email["from"]
@@ -238,10 +238,10 @@ def scan(KibbleBit, source):
"ascii", errors="replace"
)
).hexdigest()
- if KibbleBit.exists("person", sid):
+ if kibble_bit.exists("person", sid):
knowns[sender] = True
if not sender in knowns or name != sender:
- KibbleBit.append(
+ kibble_bit.append(
"person",
{
"upsert": True,
@@ -256,12 +256,12 @@ def scan(KibbleBit, source):
},
)
knowns[sender] = True
- replyTo = None
- if email["id"] in replyList:
- rt = replyList[email["id"]]
+ reply_to = None
+ if email["id"] in reply_list:
+ rt = reply_list[email["id"]]
for eml in js["emails"]:
if eml["id"] == rt:
- replyTo = getSender(eml)
+ reply_to = get_sender(eml)
print("Email was reply to %s" % sender)
jse = {
"organisation": source["organisation"],
@@ -273,14 +273,13 @@ def scan(KibbleBit, source):
"sender": sender,
"address": sender,
"subject": email["subject"],
- "replyto": replyTo,
+ "replyto": reply_to,
"ts": email["epoch"],
"id": email["id"],
"upsert": True,
}
- KibbleBit.append("email", jse)
- for sender in posters:
- no_posters += 1
+ kibble_bit.append("email", jse)
+ no_posters = len(posters)
jso = {
"organisation": source["organisation"],
@@ -292,7 +291,7 @@ def scan(KibbleBit, source):
"topics": topics,
}
# print("Indexing as %s" % dhash)
- KibbleBit.index("mailstats", dhash, jso)
+ kibble_bit.index("mailstats", dhash, jso)
month -= 1
if month <= 0:
month += 12
@@ -305,4 +304,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/travis.py b/kibble/scanners/scanners/travis.py
index 22b4975..2c7852f 100644
--- a/kibble/scanners/scanners/travis.py
+++ b/kibble/scanners/scanners/travis.py
@@ -39,44 +39,37 @@ def accepts(source):
return False
-def scanJob(KibbleBit, source, bid, token, TLD):
+def scan_job(kibble_bit, source, bid, token, TLD):
""" Scans a single job for activity """
- # NOW = int(datetime.datetime.utcnow().timestamp())
- dhash = hashlib.sha224(
- ("%s-%s-%s" % (source["organisation"], source["sourceURL"], bid)).encode(
- "ascii", errors="replace"
- )
- ).hexdigest()
-
# Get the job data
pages = 0
offset = 0
last_page = False
- oURL = "https://api.travis-ci.%s/repo/%s/builds" % (TLD, bid)
+ o_url = "https://api.travis-ci.%s/repo/%s/builds" % (TLD, bid)
# For as long as pagination makes sense...
while not last_page:
- bURL = "https://api.travis-ci.%s/repo/%s/builds?limit=100&offset=%u" % (
+ b_url = "https://api.travis-ci.%s/repo/%s/builds?limit=100&offset=%u" % (
TLD,
bid,
offset,
)
- KibbleBit.pprint("Scanning %s" % bURL)
+ kibble_bit.pprint("Scanning %s" % b_url)
rv = requests.get(
- bURL,
+ b_url,
headers={"Travis-API-Version": "3", "Authorization": "token %s" % token},
)
if rv.status_code == 200:
repojs = rv.json()
# If travis tells us it's the last page, trust it.
if repojs["@pagination"]["is_last"]:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Assuming this is the last page we need (travis says so)"
)
last_page = True
- KibbleBit.pprint(
- "%s has %u builds done" % (bURL, repojs["@pagination"]["count"])
+ kibble_bit.pprint(
+ "%s has %u builds done" % (b_url, repojs["@pagination"]["count"])
)
# BREAKER: If we go past count somehow, and travis doesn't say so, bork anyway
@@ -85,10 +78,10 @@ def scanJob(KibbleBit, source, bid, token, TLD):
offset += 100
for build in repojs.get("builds", []):
- buildID = build["id"]
- buildProject = build["repository"]["slug"]
- startedAt = build["started_at"]
- finishedAt = build["finished_at"]
+ build_id = build["id"]
+ build_project = build["repository"]["slug"]
+ started_at = build["started_at"]
+ finished_at = build["finished_at"]
duration = build["duration"]
completed = True if duration else False
duration = duration or 0
@@ -96,13 +89,13 @@ def scanJob(KibbleBit, source, bid, token, TLD):
buildhash = hashlib.sha224(
(
"%s-%s-%s-%s"
- % (source["organisation"], source["sourceURL"], bid, buildID)
+ % (source["organisation"], source["sourceURL"], bid, build_id)
).encode("ascii", errors="replace")
).hexdigest()
builddoc = None
try:
- builddoc = KibbleBit.get("ci_build", buildhash)
- except:
+ builddoc = kibble_bit.get("ci_build", buildhash)
+ except: # pylint: disable=bare-except
pass
# If this build already completed, no need to parse it again
@@ -110,7 +103,7 @@ def scanJob(KibbleBit, source, bid, token, TLD):
# If we're on page > 1 and we've seen a completed build, assume
# that we don't need the older ones
if pages > 1:
- KibbleBit.pprint(
+ kibble_bit.pprint(
"Assuming this is the last page we need (found completed build on page > 1)"
)
last_page = True
@@ -126,16 +119,16 @@ def scanJob(KibbleBit, source, bid, token, TLD):
if build["state"] in ["aborted", "canceled"]:
status = "aborted"
- FIN = 0
- STA = 0
- if finishedAt:
- FIN = datetime.datetime.strptime(
- finishedAt, "%Y-%m-%dT%H:%M:%SZ"
+ fin = 0
+ sta = 0
+ if finished_at:
+ fin = datetime.datetime.strptime(
+ finished_at, "%Y-%m-%dT%H:%M:%SZ"
).timestamp()
- if startedAt:
- STA = int(
+ if started_at:
+ sta = int(
datetime.datetime.strptime(
- startedAt, "%Y-%m-%dT%H:%M:%SZ"
+ started_at, "%Y-%m-%dT%H:%M:%SZ"
).timestamp()
)
@@ -145,14 +138,14 @@ def scanJob(KibbleBit, source, bid, token, TLD):
doc = {
# Build specific data
"id": buildhash,
- "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(FIN)),
- "buildID": buildID,
+ "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(fin)),
+ "buildID": build_id,
"completed": completed,
"duration": duration * 1000,
- "job": buildProject,
- "jobURL": oURL,
+ "job": build_project,
+ "jobURL": o_url,
"status": status,
- "started": STA,
+ "started": sta,
"ci": "travis",
"queuetime": queuetime,
# Standard docs values
@@ -160,31 +153,31 @@ def scanJob(KibbleBit, source, bid, token, TLD):
"organisation": source["organisation"],
"upsert": True,
}
- KibbleBit.append("ci_build", doc)
+ kibble_bit.append("ci_build", doc)
pages += 1
else:
# We hit a snag, abort!
- KibbleBit.pprint("Travis returned a non-200 response, aborting.")
+ kibble_bit.pprint("Travis returned a non-200 response, aborting.")
return False
return True
-class travisThread(threading.Thread):
+class TravisThread(threading.Thread):
""" Generic thread class for scheduling multiple scans at once """
- def __init__(self, block, KibbleBit, source, token, jobs, TLD):
- super(travisThread, self).__init__()
+ def __init__(self, block, kibble_bit, source, token, jobs, TLD):
+ super(TravisThread, self).__init__()
self.block = block
- self.KibbleBit = KibbleBit
+ self.kibble_bit = kibble_bit
self.token = token
self.source = source
self.jobs = jobs
self.tld = TLD
def run(self):
- badOnes = 0
- while len(self.jobs) > 0 and badOnes <= 50:
+ bad_ones = 0
+ while len(self.jobs) > 0 and bad_ones <= 50:
self.block.acquire()
try:
job = self.jobs.pop(0)
@@ -195,11 +188,11 @@ class travisThread(threading.Thread):
self.block.release()
return
self.block.release()
- if not scanJob(self.KibbleBit, self.source, job, self.token, self.tld):
- self.KibbleBit.pprint("[%s] This borked, trying another one" % job)
- badOnes += 1
- if badOnes > 100:
- self.KibbleBit.pprint("Too many errors, bailing!")
+ if not scan_job(self.kibble_bit, self.source, job, self.token, self.tld):
+ self.kibble_bit.pprint("[%s] This borked, trying another one" % job)
+ bad_ones += 1
+ if bad_ones > 100:
+ self.kibble_bit.pprint("Too many errors, bailing!")
self.source["steps"]["travis"] = {
"time": time.time(),
"status": "Too many errors while parsing at "
@@ -207,38 +200,37 @@ class travisThread(threading.Thread):
"running": False,
"good": False,
}
- self.KibbleBit.updateSource(self.source)
+ self.kibble_bit.update_source(self.source)
return
else:
- badOnes = 0
+ bad_ones = 0
-def sclan(KibbleBit, source):
+def scan(kibble_bit, source):
# Simple URL check
travis = re.match(r"https?://travis-ci\.(org|com)", source["sourceURL"])
if travis:
# Is this travs-ci.org or travis-ci.com - we need to know!
- TLD = travis.group(1)
+ tld = travis.group(1)
source["steps"]["travis"] = {
"time": time.time(),
"status": "Parsing Travis job changes...",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
- pendingJobs = []
- KibbleBit.pprint("Parsing Travis activity at %s" % source["sourceURL"])
+ pending_jobs = []
+ kibble_bit.pprint("Parsing Travis activity at %s" % source["sourceURL"])
source["steps"]["travis"] = {
"time": time.time(),
"status": "Downloading changeset",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
# Travis needs a token
- token = None
if (
source["creds"]
and "token" in source["creds"]
@@ -247,7 +239,7 @@ def sclan(KibbleBit, source):
):
token = source["creds"]["token"]
else:
- KibbleBit.pprint("Travis CI requires a token to work!")
+ kibble_bit.pprint("Travis CI requires a token to work!")
return False
# Used for pagination
@@ -262,15 +254,15 @@ def sclan(KibbleBit, source):
stuck = 0 # Ditto
avgqueuetime = 0 # Ditto, fake it
- maybeQueued = []
+ maybe_queued = []
while jobs == 100:
- URL = (
+ url = (
"https://api.travis-ci.%s/repos?repository.active=true&sort_by=current_build:desc&offset=%u&limit=100&include=repository.last_started_build"
- % (TLD, offset)
+ % (tld, offset)
)
offset += 100
r = requests.get(
- URL,
+ url,
headers={
"Travis-API-Version": "3",
"Authorization": "token %s" % token,
@@ -278,7 +270,7 @@ def sclan(KibbleBit, source):
)
if r.status_code != 200:
- KibbleBit.pprint("Travis did not return a 200 Okay, bad token?!")
+ kibble_bit.pprint("Travis did not return a 200 Okay, bad token?!")
source["steps"]["travis"] = {
"time": time.time(),
@@ -289,7 +281,7 @@ def sclan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
return
# For each build job
@@ -301,20 +293,20 @@ def sclan(KibbleBit, source):
# Is the build currently running?
if cb["state"] in ["started", "created", "queued", "pending"]:
for job in cb.get("jobs", []):
- maybeQueued.append(job["id"])
+ maybe_queued.append(job["id"])
# Queue up build jobs for the threaded scanner
bid = repo["id"]
- pendingJobs.append(bid)
+ pending_jobs.append(bid)
jobs = len(js["repositories"])
- KibbleBit.pprint("Scanned %u jobs..." % total)
+ kibble_bit.pprint("Scanned %u jobs..." % total)
# Find out how many building and pending jobs
- for jobID in maybeQueued:
- URL = "https://api.travis-ci.%s/job/%u" % (TLD, jobID)
+ for job_id in maybe_queued:
+ url = "https://api.travis-ci.%s/job/%u" % (tld, job_id)
r = requests.get(
- URL,
+ url,
headers={
"Travis-API-Version": "3",
"Authorization": "token %s" % token,
@@ -324,15 +316,15 @@ def sclan(KibbleBit, source):
jobjs = r.json()
if jobjs["state"] == "started":
building += 1
- KibbleBit.pprint("Job %u is building" % jobID)
+ kibble_bit.pprint("Job %u is building" % job_id)
elif jobjs["state"] in ["created", "queued", "pending"]:
queued += 1
blocked += 1 # Queued in Travis generally means a job can't find an executor, and thus is blocked.
- KibbleBit.pprint("Job %u is pending" % jobID)
- KibbleBit.pprint("%u building, %u queued..." % (building, queued))
+ kibble_bit.pprint("Job %u is pending" % job_id)
+ kibble_bit.pprint("%u building, %u queued..." % (building, queued))
# Save queue snapshot
- NOW = int(datetime.datetime.utcnow().timestamp())
+ now = int(datetime.datetime.utcnow().timestamp())
queuehash = hashlib.sha224(
(
"%s-%s-queue-%s"
@@ -343,8 +335,8 @@ def sclan(KibbleBit, source):
# Write up a queue doc
queuedoc = {
"id": queuehash,
- "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(NOW)),
- "time": NOW,
+ "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(now)),
+ "time": now,
"building": building,
"size": queued,
"blocked": blocked,
@@ -356,15 +348,15 @@ def sclan(KibbleBit, source):
"organisation": source["organisation"],
"upsert": True,
}
- KibbleBit.append("ci_queue", queuedoc)
+ kibble_bit.append("ci_queue", queuedoc)
- KibbleBit.pprint("Found %u jobs in Travis" % len(pendingJobs))
+ kibble_bit.pprint("Found %u jobs in Travis" % len(pending_jobs))
threads = []
block = threading.Lock()
- KibbleBit.pprint("Scanning jobs using 4 sub-threads")
+ kibble_bit.pprint("Scanning jobs using 4 sub-threads")
for i in range(0, 4):
- t = travisThread(block, KibbleBit, source, token, pendingJobs, TLD)
+ t = TravisThread(block, kibble_bit, source, token, pending_jobs, tld)
threads.append(t)
t.start()
@@ -372,7 +364,7 @@ def sclan(KibbleBit, source):
t.join()
# We're all done, yaay
- KibbleBit.pprint("Done scanning %s" % source["sourceURL"])
+ kibble_bit.pprint("Done scanning %s" % source["sourceURL"])
source["steps"]["travis"] = {
"time": time.time(),
@@ -381,4 +373,4 @@ def sclan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
diff --git a/kibble/scanners/scanners/twitter.py b/kibble/scanners/scanners/twitter.py
index fd6af5b..dc87ed2 100644
--- a/kibble/scanners/scanners/twitter.py
+++ b/kibble/scanners/scanners/twitter.py
@@ -37,7 +37,7 @@ def accepts(source):
return False
-def getFollowers(KibbleBit, source, t):
+def get_followers(kibble_bit, source, t):
""" Get followers of a handle, store them for mapping and trend purposes"""
# Get our twitter handle
handle = source["sourceURL"]
@@ -59,18 +59,18 @@ def getFollowers(KibbleBit, source, t):
"followers": no_followers,
"date": d,
}
- KibbleBit.pprint("%s has %u followers currently." % (handle, no_followers))
- KibbleBit.index("twitter_followers", dhash, jst)
+ kibble_bit.pprint("%s has %u followers currently." % (handle, no_followers))
+ kibble_bit.index("twitter_followers", dhash, jst)
# Collect list of current followers
followers = t.GetFollowers(screen_name=handle)
# For each follower, if they're not mapped yet, add them
# This has a limitation of 100 new added per run, but meh...
- KibbleBit.pprint("Looking up followers of %s" % handle)
+ kibble_bit.pprint("Looking up followers of %s" % handle)
for follower in followers:
# id, name, screen_name are useful here
- KibbleBit.pprint("Found %s as follower" % follower.screen_name)
+ kibble_bit.pprint("Found %s as follower" % follower.screen_name)
# Store twitter follower profile if not already logged
dhash = hashlib.sha224(
@@ -78,7 +78,7 @@ def getFollowers(KibbleBit, source, t):
"ascii", errors="replace"
)
).hexdigest()
- if not KibbleBit.exists("twitter_follow", dhash):
+ if not kibble_bit.exists("twitter_follow", dhash):
jst = {
"organisation": source["organisation"],
"sourceURL": source["sourceURL"],
@@ -91,20 +91,20 @@ def getFollowers(KibbleBit, source, t):
"%Y/%m/%d %H:%M:%S", time.gmtime()
), # First time we spotted them following.
}
- KibbleBit.pprint(
+ kibble_bit.pprint(
"%s is new, recording date and details." % follower.screen_name
)
- KibbleBit.index("twitter_follow", dhash, jst)
+ kibble_bit.index("twitter_follow", dhash, jst)
-def scan(KibbleBit, source):
+def scan(kibble_bit, source):
source["steps"]["twitter"] = {
"time": time.time(),
"status": "Scanning Twitter activity and status",
"running": True,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)
t = None
if "creds" in source and source["creds"]:
t = twitter.Api(
@@ -113,22 +113,22 @@ def scan(KibbleBit, source):
consumer_key=source["creds"].get("consumer_key", None),
consumer_secret=source["creds"].get("consumer_secret", None),
)
- KibbleBit.pprint("Verifying twitter credentials...")
+ kibble_bit.pprint("Verifying twitter credentials...")
try:
t.VerifyCredentials()
- except:
+ except: # pylint: disable=bare-except
source["steps"]["twitter"] = {
"time": time.time(),
"status": "Could not verify twitter credentials",
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
- KibbleBit.pprint("Could not verify twitter creds, aborting!")
+ kibble_bit.update_source(source)
+ kibble_bit.pprint("Could not verify twitter creds, aborting!")
return
# Start by getting and saving followers
try:
- getFollowers(KibbleBit, source, t)
+ get_followers(kibble_bit, source, t)
except Exception as err:
source["steps"]["twitter"] = {
"time": time.time(),
@@ -136,8 +136,8 @@ def scan(KibbleBit, source):
"running": False,
"good": False,
}
- KibbleBit.updateSource(source)
- KibbleBit.pprint("Twitter scan failed: %s" % err)
+ kibble_bit.update_source(source)
+ kibble_bit.pprint("Twitter scan failed: %s" % err)
# All done, report that!
source["steps"]["twitter"] = {
@@ -147,4 +147,4 @@ def scan(KibbleBit, source):
"running": False,
"good": True,
}
- KibbleBit.updateSource(source)
+ kibble_bit.update_source(source)