You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kibble.apache.org by hu...@apache.org on 2018/02/16 18:56:17 UTC
[kibble] 02/05: Add preliminary CI status pages
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git
commit 2fd2c89040320f8216864e3f21e09d20c5bbe159
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Fri Feb 16 19:55:40 2018 +0100
Add preliminary CI status pages
---
api/pages/ci/queue.py | 182 ++++++++++++++++++++++++++++++++++++++++
api/pages/ci/status.py | 180 ++++++++++++++++++++++++++++++++++++++++
api/pages/ci/top-buildcount.py | 183 +++++++++++++++++++++++++++++++++++++++++
api/pages/ci/top-buildtime.py | 183 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 728 insertions(+)
diff --git a/api/pages/ci/queue.py b/api/pages/ci/queue.py
new file mode 100644
index 0000000..06d495d
--- /dev/null
+++ b/api/pages/ci/queue.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################
+# OPENAPI-URI: /api/ci/queue
+########################################################################
+# get:
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows email sent over time
+# post:
+# requestBody:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/defaultWidgetArgs'
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows CI queue over time
+#
+########################################################################
+
+
+
+"""
+This is the CI queue timeseries renderer for Kibble
+"""
+
+import json
+import time
+import hashlib
+
+def run(API, environ, indata, session):
+
+ # We need to be logged in for this!
+ if not session.user:
+ raise API.exception(403, "You must be logged in to use this API endpoint! %s")
+
+ now = time.time()
+
+ # First, fetch the view if we have such a thing enabled
+ viewList = []
+ if indata.get('view'):
+ viewList = session.getView(indata.get('view'))
+ if indata.get('subfilter'):
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
+ dateTo = indata.get('to', int(time.time()))
+ dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
+
+ interval = indata.get('interval', 'month')
+
+
+ ####################################################################
+ ####################################################################
+ dOrg = session.user['defaultOrganisation'] or "apache"
+ query = {
+ 'query': {
+ 'bool': {
+ 'must': [
+ {'range':
+ {
+ 'time': {
+ 'from': dateFrom,
+ 'to': dateTo
+ }
+ }
+ },
+ {
+ 'term': {
+ 'organisation': dOrg
+ }
+ }
+ ]
+ }
+ }
+ }
+ # Source-specific or view-specific??
+ if indata.get('source'):
+ query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
+ elif viewList:
+ query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
+
+ # Get queue stats
+ query['aggs'] = {
+ 'timeseries': {
+ 'date_histogram': {
+ 'field': 'date',
+ 'interval': interval
+ },
+ 'aggs': {
+ 'size': {
+ 'avg': {
+ 'field': 'size'
+ }
+ },
+ 'blocked': {
+ 'avg': {
+ 'field': 'blocked'
+ }
+ },
+ 'stuck': {
+ 'avg': {
+ 'field': 'stuck'
+ }
+ },
+ 'wait': {
+ 'avg': {
+ 'field': 'avgwait'
+ }
+ }
+ }
+ }
+ }
+ res = session.DB.ES.search(
+ index=session.DB.dbname,
+ doc_type="ci_queue",
+ size = 0,
+ body = query
+ )
+
+ timeseries = []
+ for bucket in res['aggregations']['timeseries']['buckets']:
+ ts = int(bucket['key'] / 1000)
+ timeseries.append({
+ 'date': ts,
+ 'queue size': bucket['size']['value'],
+# 'builds blocked': bucket['blocked']['value'],
+# 'builds stuck': bucket['stuck']['value'],
+ 'average wait (hours)': int(bucket['wait']['value']/3600),
+ })
+
+ JSON_OUT = {
+ 'widgetType': {
+ 'chartType': 'line' # Recommendation for the UI
+ },
+ 'timeseries': timeseries,
+ 'interval': interval,
+ 'okay': True,
+ 'responseTime': time.time() - now
+ }
+ yield json.dumps(JSON_OUT)
diff --git a/api/pages/ci/status.py b/api/pages/ci/status.py
new file mode 100644
index 0000000..e3678a2
--- /dev/null
+++ b/api/pages/ci/status.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################
+# OPENAPI-URI: /api/ci/status
+########################################################################
+# get:
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows email sent over time
+# post:
+# requestBody:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/defaultWidgetArgs'
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows CI queue over time
+#
+########################################################################
+
+
+
+"""
+This is the CI queue status (blocked/stuck) timeseries renderer for Kibble
+"""
+
+import json
+import time
+import hashlib
+
+def run(API, environ, indata, session):
+
+ # We need to be logged in for this!
+ if not session.user:
+ raise API.exception(403, "You must be logged in to use this API endpoint! %s")
+
+ now = time.time()
+
+ # First, fetch the view if we have such a thing enabled
+ viewList = []
+ if indata.get('view'):
+ viewList = session.getView(indata.get('view'))
+ if indata.get('subfilter'):
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+
+ dateTo = indata.get('to', int(time.time()))
+ dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
+
+ interval = indata.get('interval', 'month')
+
+
+ ####################################################################
+ ####################################################################
+ dOrg = session.user['defaultOrganisation'] or "apache"
+ query = {
+ 'query': {
+ 'bool': {
+ 'must': [
+ {'range':
+ {
+ 'time': {
+ 'from': dateFrom,
+ 'to': dateTo
+ }
+ }
+ },
+ {
+ 'term': {
+ 'organisation': dOrg
+ }
+ }
+ ]
+ }
+ }
+ }
+ # Source-specific or view-specific??
+ if indata.get('source'):
+ query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
+ elif viewList:
+ query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
+
+ # Get queue stats
+ query['aggs'] = {
+ 'timeseries': {
+ 'date_histogram': {
+ 'field': 'date',
+ 'interval': interval
+ },
+ 'aggs': {
+ 'size': {
+ 'avg': {
+ 'field': 'size'
+ }
+ },
+ 'blocked': {
+ 'avg': {
+ 'field': 'blocked'
+ }
+ },
+ 'stuck': {
+ 'avg': {
+ 'field': 'stuck'
+ }
+ },
+ 'wait': {
+ 'avg': {
+ 'field': 'avgwait'
+ }
+ }
+ }
+ }
+ }
+ res = session.DB.ES.search(
+ index=session.DB.dbname,
+ doc_type="ci_queue",
+ size = 0,
+ body = query
+ )
+
+ timeseries = []
+ for bucket in res['aggregations']['timeseries']['buckets']:
+ ts = int(bucket['key'] / 1000)
+ timeseries.append({
+ 'date': ts,
+ 'builds blocked': bucket['blocked']['value'],
+ 'builds stuck': bucket['stuck']['value']
+ })
+
+ JSON_OUT = {
+ 'widgetType': {
+ 'chartType': 'bar' # Recommendation for the UI
+ },
+ 'timeseries': timeseries,
+ 'interval': interval,
+ 'okay': True,
+ 'responseTime': time.time() - now
+ }
+ yield json.dumps(JSON_OUT)
diff --git a/api/pages/ci/top-buildcount.py b/api/pages/ci/top-buildcount.py
new file mode 100644
index 0000000..31ef5c3
--- /dev/null
+++ b/api/pages/ci/top-buildcount.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################
+# OPENAPI-URI: /api/ci/top-buildcount
+########################################################################
+# get:
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows top 25 repos by lines of code
+# post:
+# requestBody:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/defaultWidgetArgs'
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows top 25 jobs by total builds done. Essentially buildtime, tweaked
+#
+########################################################################
+
+
+
+
+
+"""
+This is the TopN CI jobs by total build time renderer for Kibble
+"""
+
+import json
+import time
+import re
+
+def run(API, environ, indata, session):
+
+ # We need to be logged in for this!
+ if not session.user:
+ raise API.exception(403, "You must be logged in to use this API endpoint! %s")
+
+ now = time.time()
+
+ # First, fetch the view if we have such a thing enabled
+ viewList = []
+ if indata.get('view'):
+ viewList = session.getView(indata.get('view'))
+ if indata.get('subfilter'):
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+ dateTo = indata.get('to', int(time.time()))
+ dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
+
+ ####################################################################
+ ####################################################################
+ dOrg = session.user['defaultOrganisation'] or "apache"
+ query = {
+ 'query': {
+ 'bool': {
+ 'must': [
+ {'range':
+ {
+ 'date': {
+ 'from': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateFrom)),
+ 'to': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateTo))
+ }
+ }
+ },
+ {
+ 'term': {
+ 'organisation': dOrg
+ }
+ }
+ ]
+ }
+ }
+ }
+ # Source-specific or view-specific??
+ if indata.get('source'):
+ query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
+ elif viewList:
+ query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
+
+ query['aggs'] = {
+ 'by_job': {
+ 'terms': {
+ 'field': 'jobURL.keyword',
+ 'size': 5000,
+ },
+ 'aggs': {
+ 'duration': {
+ 'sum': {
+ 'field': 'duration'
+ }
+ },
+ 'ci': {
+ 'terms': {
+ 'field': 'ci.keyword',
+ 'size': 1
+ }
+ },
+ 'name': {
+ 'terms': {
+ 'field': 'job.keyword',
+ 'size': 1
+ }
+ }
+ }
+ }
+ }
+
+ res = session.DB.ES.search(
+ index=session.DB.dbname,
+ doc_type="ci_build",
+ size = 0,
+ body = query
+ )
+
+ jobs = []
+ for doc in res['aggregations']['by_job']['buckets']:
+ job = doc['key']
+ builds = doc['doc_count']
+ duration = doc['duration']['value']
+ ci = doc['ci']['buckets'][0]['key']
+ jobname = doc['name']['buckets'][0]['key']
+ jobs.append([builds, duration, jobname, ci])
+
+ topjobs = sorted(jobs, key = lambda x: int(x[0]), reverse = True)
+ top = topjobs[0:24]
+ if len(topjobs) > 25:
+ count = 0
+ for repo in topjobs[25:]:
+ count += repo[1]
+ top.append(["Other jobs", 1, count, '??'])
+
+ tophash = {}
+ for v in top:
+ tophash["%s (%s)" % (v[2], v[3])] = v[0]
+
+ JSON_OUT = {
+ 'counts': tophash,
+ 'okay': True,
+ 'responseTime': time.time() - now,
+ }
+ yield json.dumps(JSON_OUT)
diff --git a/api/pages/ci/top-buildtime.py b/api/pages/ci/top-buildtime.py
new file mode 100644
index 0000000..995bd2c
--- /dev/null
+++ b/api/pages/ci/top-buildtime.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################
+# OPENAPI-URI: /api/ci/top-buildtime
+########################################################################
+# get:
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows top 25 repos by lines of code
+# post:
+# requestBody:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/defaultWidgetArgs'
+# responses:
+# '200':
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Timeseries'
+# description: 200 Response
+# default:
+# content:
+# application/json:
+# schema:
+# $ref: '#/components/schemas/Error'
+# description: unexpected error
+# security:
+# - cookieAuth: []
+# summary: Shows top 25 jobs by total build time spent
+#
+########################################################################
+
+
+
+
+
+"""
+This is the TopN CI jobs by total build time renderer for Kibble
+"""
+
+import json
+import time
+import re
+
+def run(API, environ, indata, session):
+
+ # We need to be logged in for this!
+ if not session.user:
+ raise API.exception(403, "You must be logged in to use this API endpoint! %s")
+
+ now = time.time()
+
+ # First, fetch the view if we have such a thing enabled
+ viewList = []
+ if indata.get('view'):
+ viewList = session.getView(indata.get('view'))
+ if indata.get('subfilter'):
+ viewList = session.subFilter(indata.get('subfilter'), view = viewList)
+
+ dateTo = indata.get('to', int(time.time()))
+ dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span
+
+ ####################################################################
+ ####################################################################
+ dOrg = session.user['defaultOrganisation'] or "apache"
+ query = {
+ 'query': {
+ 'bool': {
+ 'must': [
+ {'range':
+ {
+ 'date': {
+ 'from': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateFrom)),
+ 'to': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateTo))
+ }
+ }
+ },
+ {
+ 'term': {
+ 'organisation': dOrg
+ }
+ }
+ ]
+ }
+ }
+ }
+ # Source-specific or view-specific??
+ if indata.get('source'):
+ query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}})
+ elif viewList:
+ query['query']['bool']['must'].append({'terms': {'sourceID': viewList}})
+
+ query['aggs'] = {
+ 'by_job': {
+ 'terms': {
+ 'field': 'jobURL.keyword',
+ 'size': 5000,
+ },
+ 'aggs': {
+ 'duration': {
+ 'sum': {
+ 'field': 'duration'
+ }
+ },
+ 'ci': {
+ 'terms': {
+ 'field': 'ci.keyword',
+ 'size': 1
+ }
+ },
+ 'name': {
+ 'terms': {
+ 'field': 'job.keyword',
+ 'size': 1
+ }
+ }
+ }
+ }
+ }
+
+ res = session.DB.ES.search(
+ index=session.DB.dbname,
+ doc_type="ci_build",
+ size = 0,
+ body = query
+ )
+
+ jobs = []
+ for doc in res['aggregations']['by_job']['buckets']:
+ job = doc['key']
+ builds = doc['doc_count']
+ duration = doc['duration']['value']
+ ci = doc['ci']['buckets'][0]['key']
+ jobname = doc['name']['buckets'][0]['key']
+ jobs.append([builds, duration, jobname, ci])
+
+ topjobs = sorted(jobs, key = lambda x: int(x[1]), reverse = True)
+ top = topjobs[0:24]
+ if len(topjobs) > 25:
+ count = 0
+ for repo in topjobs[25:]:
+ count += repo[1]
+ top.append(["Other jobs", 1, count, '??'])
+
+ tophash = {}
+ for v in top:
+ tophash["%s (%s)" % (v[2], v[3])] = int((v[1]/86400))
+
+ JSON_OUT = {
+ 'counts': tophash,
+ 'okay': True,
+ 'responseTime': time.time() - now,
+ }
+ yield json.dumps(JSON_OUT)
--
To stop receiving notification emails like this one, please contact
humbedooh@apache.org.