You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2019/12/02 22:36:26 UTC
[trafficcontrol] branch master updated: Rewrite current stats from
Perl to Golang (#4114)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 752ee5a Rewrite current stats from Perl to Golang (#4114)
752ee5a is described below
commit 752ee5a73cecb81ba8c13f26e7b6d814b26ea9a3
Author: Michael Hoppal <54...@users.noreply.github.com>
AuthorDate: Mon Dec 2 15:36:17 2019 -0700
Rewrite current stats from Perl to Golang (#4114)
* Rewrite current stats from Perl to Golang
* feedback on PR
* Match perl response
* Fix Docs
---
CHANGELOG.md | 1 +
docs/source/api/current_stats.rst | 92 +++++++++++
lib/go-tc/traffic_stats.go | 26 ++++
traffic_ops/client/traffic_stats.go | 29 ++++
traffic_ops/traffic_ops_golang/routing/routes.go | 5 +-
traffic_ops/traffic_ops_golang/trafficstats/cdn.go | 168 +++++++++++++++++++++
6 files changed, 319 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8cd68c9..0f446ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- /api/1.1/deliveryservices/request
- /api/1.1/federations/:id/users
- /api/1.1/federations/:id/users/:userID
+ - /api/1.2/current_stats
- /api/1.1/osversions
- To support reusing a single riak cluster connection, an optional parameter is added to riak.conf: "HealthCheckInterval". This options takes a 'Duration' value (ie: 10s, 5m) which affects how often the riak cluster is health checked. Default is currently set to: "HealthCheckInterval": "5s".
diff --git a/docs/source/api/current_stats.rst b/docs/source/api/current_stats.rst
new file mode 100644
index 0000000..651fe14
--- /dev/null
+++ b/docs/source/api/current_stats.rst
@@ -0,0 +1,92 @@
+..
+..
+.. Licensed under the Apache License, Version 2.0 (the "License");
+.. you may not use this file except in compliance with the License.
+.. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+..
+
+
+.. _to-api-current-stats:
+
+*****************
+``current_stats``
+*****************
+An API endpoint that returns current statistics for each CDN and an aggregate across them all.
+
+.. versionadded:: 1.2
+
+``GET``
+=======
+Retrieves current stats for each CDN. Also includes aggregate stats across them.
+
+:Auth. Required: Yes
+:Roles Required: None
+:Response Type: Array
+
+Request Structure
+-----------------
+No parameters available.
+
+.. code-block:: http
+ :caption: Request Example
+
+ GET /api/1.4/current_stats HTTP/1.1
+ Host: trafficops.infra.ciab.test
+ User-Agent: curl/7.47.0
+ Accept: */*
+ Cookie: mojolicious=...
+
+Response Structure
+------------------
+:cdn: The name of the CDN
+:connections: Current number of TCP connections maintained
+:capacity: 85 percent capacity of the CDN in Gbps
+:bandwidth: The total amount of bandwidth in Gbs
+
+.. note:: If ``cdn`` name is total and capacity is omitted it represents the aggregate stats across CDNs
+
+.. code-block:: http
+ :caption: Response Example
+
+ HTTP/1.1 200 OK
+ Access-Control-Allow-Credentials: true
+ Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Set-Cookie, Cookie
+ Access-Control-Allow-Methods: POST,GET,OPTIONS,PUT,DELETE
+ Access-Control-Allow-Origin: *
+ Content-Encoding: gzip
+ Content-Type: application/json
+ Set-Cookie: mojolicious=; Path=/; HttpOnly
+ Whole-Content-Sha512: Rs3wgd7v5dP0bOQs4I3J1q6mnWIMSM2AKSAWirK1kymvDYOoFISArF7Kkypgy10I34yn7FtFdMh6U7ABaS1Tjw==
+ X-Server-Name: traffic_ops_golang/
+ Date: Thu, 14 Nov 2019 15:35:31 GMT
+ Content-Length: 138
+
+ {"response": {
+ "currentStats": [
+ {
+ "bandwidth": null,
+ "capacity": null,
+ "cdn": "ALL",
+ "connections": null
+ },
+ {
+ "bandwidth": 0.000104,
+ "capacity": 17,
+ "cdn": "CDN-in-a-Box",
+ "connections": 4
+ },
+ {
+ "bandwidth": 0.000104,
+ "cdn": "total",
+ "connections": 4
+ }
+ ]
+ }}
diff --git a/lib/go-tc/traffic_stats.go b/lib/go-tc/traffic_stats.go
index ca0a34d..2ca469b 100644
--- a/lib/go-tc/traffic_stats.go
+++ b/lib/go-tc/traffic_stats.go
@@ -270,3 +270,29 @@ func MessagesToString(msgs []influx.Message) string {
b.WriteRune(']')
return b.String()
}
+
+// TrafficStatsCDNStats contains summary statistics for a given CDN
+type TrafficStatsCDNStats struct {
+ Bandwidth *float64 `json:"bandwidth"`
+ Capacity *float64 `json:"capacity"`
+ CDN string `json:"cdn"`
+ Connnections *float64 `json:"connections"`
+}
+
+// TrafficStatsTotalStats contains summary statistics across CDNs
+// Different then TrafficStatsCDNStats as it omits Capacity
+type TrafficStatsTotalStats struct {
+ Bandwidth *float64 `json:"bandwidth"`
+ CDN string `json:"cdn"`
+ Connnections *float64 `json:"connections"`
+}
+
+// TrafficStatsCDNStatsResponse contains response for getting current stats
+type TrafficStatsCDNStatsResponse struct {
+ Response []TrafficStatsCDNsStats `json:"response"`
+}
+
+// TrafficStatsCDNsStats contains a list of CDN summary statistics
+type TrafficStatsCDNsStats struct {
+ Stats []TrafficStatsCDNStats `json:"currentStats"`
+}
diff --git a/traffic_ops/client/traffic_stats.go b/traffic_ops/client/traffic_stats.go
new file mode 100644
index 0000000..8f163d9
--- /dev/null
+++ b/traffic_ops/client/traffic_stats.go
@@ -0,0 +1,29 @@
+package client
+
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import (
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+// GetCurrentStats gets current stats for each CDNs and a total across them
+func (to *Session) GetCurrentStats() (tc.TrafficStatsCDNStatsResponse, ReqInf, error) {
+ resp := tc.TrafficStatsCDNStatsResponse{}
+ reqInf, err := get(to, apiBase+"/current_stats", &resp)
+ if err != nil {
+ return resp, reqInf, err
+ }
+ return resp, reqInf, nil
+}
diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go
index 63cb1c8..ff6cfd9 100644
--- a/traffic_ops/traffic_ops_golang/routing/routes.go
+++ b/traffic_ops/traffic_ops_golang/routing/routes.go
@@ -117,8 +117,9 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) {
{1.1, http.MethodDelete, `asns/{id}$`, api.DeleteHandler(&asn.TOASNV11{}), auth.PrivLevelOperations, Authenticated, nil},
// Traffic Stats access
- {1.2, http.MethodGet, `deliveryservice_stats`, trafficstats.GetDSStats, auth.PrivLevelReadOnly, Authenticated, nil},
- {1.2, http.MethodGet, `cache_stats`, trafficstats.GetCacheStats, auth.PrivLevelReadOnly, Authenticated, nil},
+ {1.2, http.MethodGet, `deliveryservice_stats/?(\.json)?$`, trafficstats.GetDSStats, auth.PrivLevelReadOnly, Authenticated, nil},
+ {1.2, http.MethodGet, `cache_stats/?(\.json)?$`, trafficstats.GetCacheStats, auth.PrivLevelReadOnly, Authenticated, nil},
+ {1.2, http.MethodGet, `current_stats/?(\.json)?$`, trafficstats.GetCurrentStats, auth.PrivLevelReadOnly, Authenticated, nil},
{1.1, http.MethodGet, `caches/stats/?(\.json)?$`, cachesstats.Get, auth.PrivLevelReadOnly, Authenticated, nil},
diff --git a/traffic_ops/traffic_ops_golang/trafficstats/cdn.go b/traffic_ops/traffic_ops_golang/trafficstats/cdn.go
new file mode 100644
index 0000000..04fa902
--- /dev/null
+++ b/traffic_ops/traffic_ops_golang/trafficstats/cdn.go
@@ -0,0 +1,168 @@
+package trafficstats
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+
+ influx "github.com/influxdata/influxdb/client/v2"
+ "github.com/lib/pq"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/lib/go-util"
+ "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
+)
+
+const (
+ cdnStatsQuery = `
+SELECT last(value) FROM "%s"."monthly"."%s"
+ WHERE cdn = $cdn`
+ bwMetricName = "bandwidth.cdn.1min"
+ connMetricName = "connections.cdn.1min"
+ kbpsMetricName = "maxkbps.cdn.1min"
+)
+
+// GetCurrentStats handler for getting current stats for CDNs
+func GetCurrentStats(w http.ResponseWriter, r *http.Request) {
+ inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil)
+ tx := inf.Tx.Tx
+
+ if userErr != nil || sysErr != nil {
+ api.HandleErr(w, r, tx, errCode, userErr, sysErr)
+ return
+ }
+ defer inf.Close()
+
+ client, err := inf.CreateInfluxClient()
+ if err != nil {
+ api.HandleErr(w, r, tx, http.StatusInternalServerError, nil, err)
+ return
+ } else if client == nil {
+ sysErr = errors.New("Traffic Stats is not configured, but DS stats were requested")
+ api.HandleErr(w, r, tx, http.StatusInternalServerError, nil, sysErr)
+ return
+ }
+ defer (*client).Close()
+
+ currentStats := []interface{}{}
+
+ // Get CDN names
+ cdns := []string{}
+ if err := tx.QueryRow(`SELECT ARRAY(SELECT name FROM cdn)`).Scan(pq.Array(&cdns)); err != nil {
+ api.HandleErr(w, r, tx, http.StatusInternalServerError, nil, errors.New("querying cdn names"))
+ return
+ }
+ totalStats := tc.TrafficStatsTotalStats{
+ CDN: "total",
+ }
+
+ for _, cdn := range cdns {
+ cdnStats := tc.TrafficStatsCDNStats{
+ CDN: cdn,
+ }
+ bw, err := getCDNStat(client, cdn, bwMetricName, inf.Config.ConfigInflux.CacheDBName)
+ if err != nil {
+ sysErr = fmt.Errorf("getting bandwidth from cdn %v: %v", cdn, err)
+ api.HandleErr(w, r, tx, http.StatusInternalServerError, nil, sysErr)
+ return
+ }
+ if bw != nil {
+ if totalStats.Bandwidth == nil {
+ totalStats.Bandwidth = util.FloatPtr(0.0)
+ }
+ *totalStats.Bandwidth += *bw
+ }
+
+ if bw != nil {
+ cdnStats.Bandwidth = util.FloatPtr(*bw / 1000000)
+ }
+
+ con, err := getCDNStat(client, cdn, connMetricName, inf.Config.ConfigInflux.CacheDBName)
+ if err != nil {
+ sysErr = fmt.Errorf("getting connections from cdn %v: %v", cdn, err)
+ api.HandleErr(w, r, tx, http.StatusInternalServerError, nil, sysErr)
+ return
+ }
+ if con != nil {
+ if totalStats.Connnections == nil {
+ totalStats.Connnections = util.FloatPtr(0.0)
+ }
+ *totalStats.Connnections += *con
+ }
+ cdnStats.Connnections = con
+
+ cap, err := getCDNStat(client, cdn, kbpsMetricName, inf.Config.ConfigInflux.CacheDBName)
+ if err != nil {
+ sysErr = fmt.Errorf("getting maxkbps from cdn %v: %v", cdn, err)
+ api.HandleErr(w, r, tx, http.StatusInternalServerError, nil, sysErr)
+ return
+ }
+ if cap != nil {
+ // Perl implementation hardcoded capacity as 85 percent of Gbps
+ cdnStats.Capacity = util.FloatPtr(*cap / 1000000 * .85)
+ }
+
+ currentStats = append(currentStats, cdnStats)
+ }
+
+ if totalStats.Bandwidth != nil {
+ *totalStats.Bandwidth /= 1000000
+ }
+ currentStats = append(currentStats, totalStats)
+ resp := struct {
+ CurrentStats []interface{} `json:"currentStats"`
+ }{
+ CurrentStats: currentStats,
+ }
+
+ api.WriteResp(w, r, resp)
+}
+
+func getCDNStat(client *influx.Client, cdnName, metricName, db string) (*float64, error) {
+ qStr := fmt.Sprintf(cdnStatsQuery, db, metricName)
+ q := influx.NewQueryWithParameters(qStr,
+ db,
+ "rfc3339",
+ map[string]interface{}{
+ "cdn": cdnName,
+ })
+ series, err := getSeries(db, q, client)
+ if err != nil {
+ return nil, err
+ }
+ if series == nil {
+ return nil, nil
+ }
+ if len(series.Values) == 0 {
+ return nil, fmt.Errorf("influxdb query for metrtic %v returned series with no values", metricName)
+ }
+ vals := series.Values[0]
+ mappedValues := map[string]interface{}{}
+ for i, v := range vals {
+ mappedValues[series.Columns[i]] = v
+ }
+ val, err := extractFloat64("last", mappedValues)
+ if err != nil {
+ return nil, err
+ }
+ return &val, nil
+}