[GitHub] [trafficcontrol] mitchell852 commented on a change in pull request #2306: Add TO Go cdns/capacity

[GitBox <gi...@apache.org>]

mitchell852 commented on a change in pull request #2306: Add TO Go cdns/capacity
URL: https://github.com/apache/trafficcontrol/pull/2306#discussion_r314894104
 
 

 ##########
 File path: traffic_ops/traffic_ops_golang/cdn/capacity.go
 ##########
 @@ -0,0 +1,325 @@
+package cdn
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/apache/trafficcontrol/lib/go-tc"
+	"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
+)
+
+func GetCapacity(w http.ResponseWriter, r *http.Request) {
+	inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil)
+	if userErr != nil || sysErr != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr)
+		return
+	}
+	defer inf.Close()
+
+	api.RespWriter(w, r, inf.Tx.Tx)(getCapacity(inf.Tx.Tx))
+}
+
+const MonitorProxyParameter = "tm.traffic_mon_fwd_proxy"
+const MonitorRequestTimeout = time.Second * 10
+const MonitorOnlineStatus = "ONLINE"
+
+// CRStates contains the Monitor CRStates members needed for health. It is NOT the full object served by the Monitor, but only the data required by this endpoint.
+type CRStates struct {
+	Caches map[tc.CacheName]Available `json:"caches"`
+}
+
+type Available struct {
+	IsAvailable bool `json:"isAvailable"`
+}
+
+// CRConfig contains the Monitor CRConfig members needed for health. It is NOT the full object served by the Monitor, but only the data required by this endpoint.
+type CRConfig struct {
+	ContentServers map[tc.CacheName]CRConfigServer `json:"contentServers"`
+}
+
+type CRConfigServer struct {
+	CacheGroup tc.CacheGroupName `json:"locationId"`
+	Status     tc.CacheStatus    `json:"status"`
+	Type       tc.CacheType      `json:"type"`
+	Profile    string            `json:"profile"`
+}
+
+func getCapacity(tx *sql.Tx) (CapacityResp, error) {
+	monitors, err := getCDNMonitorFQDNs(tx)
+	if err != nil {
+		return CapacityResp{}, errors.New("getting monitors: " + err.Error())
+	}
+	return getMonitorsCapacity(tx, monitors)
+}
+
+type CapacityResp struct {
+	AvailablePercent   float64 `json:"availablePercent"`
+	UnavailablePercent float64 `json:"unavailablePercent"`
+	UtilizedPercent    float64 `json:utilizedPercent"`
+	MaintenancePercent float64 `json:maintenancePercent"`
+}
+
+type CapData struct {
+	Available   float64
+	Unavailable float64
+	Utilized    float64
+	Maintenance float64
+	Capacity    float64
+}
+
+func getMonitorsCapacity(tx *sql.Tx, monitors map[tc.CDNName]string) (CapacityResp, error) {
+	monitorForwardProxy, monitorForwardProxyExists, err := getGlobalParam(tx, MonitorProxyParameter)
+	if err != nil {
+		return CapacityResp{}, errors.New("getting global monitor proxy parameter: " + err.Error())
+	}
+	client := &http.Client{Timeout: MonitorRequestTimeout}
+	if monitorForwardProxyExists {
+		proxyURI, err := url.Parse(monitorForwardProxy)
+		if err != nil {
+			return CapacityResp{}, errors.New("monitor forward proxy '" + monitorForwardProxy + "' in parameter '" + MonitorProxyParameter + "' not a URI: " + err.Error())
+		}
+		client = &http.Client{Timeout: MonitorRequestTimeout, Transport: &http.Transport{Proxy: http.ProxyURL(proxyURI)}}
+	}
+
+	thresholds, err := getEdgeProfileHealthThresholdBandwidth(tx)
+	if err != nil {
+		return CapacityResp{}, errors.New("getting profile thresholds: " + err.Error())
+	}
+
+	cap := CapData{}
+	for cdn, monitorFQDN := range monitors {
+		crStates, err := getCRStates(monitorFQDN, client)
+		// TODO on err, try another online monitor
+		if err != nil {
+			return CapacityResp{}, errors.New("getting CRStates for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error())
+		}
+		crConfig, err := getCRConfig(monitorFQDN, client)
+		// TODO on err, try another online monitor
+		if err != nil {
+			return CapacityResp{}, errors.New("getting CRConfig for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error())
+		}
+		cacheStats := CacheStats{}
+		// TODO on err, try another online monitor
+		if err := getCacheStats(monitorFQDN, client, []string{"kbps", "maxKbps"}, &cacheStats); err != nil {
+			return CapacityResp{}, errors.New("getting cache stats for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error())
+		}
+
+		cap = addCapacity(cap, cacheStats, crStates, crConfig, thresholds)
+	}
+	if cap.Capacity == 0 {
+		return CapacityResp{}, errors.New("capacity was zero!") // avoid divide-by-zero below.
+	}
+	resp := CapacityResp{}
+	resp.UtilizedPercent = (cap.Available * 100) / cap.Capacity
+	resp.UnavailablePercent = (cap.Unavailable * 100) / cap.Capacity
+	resp.MaintenancePercent = (cap.Maintenance * 100) / cap.Capacity
+	resp.AvailablePercent = ((cap.Capacity - cap.Unavailable - cap.Maintenance - cap.Available) * 100) / cap.Capacity
+	return resp, nil
+}
+
+func addCapacity(cap CapData, cacheStats CacheStats, crStates CRStates, crConfig CRConfig, thresholds map[string]float64) CapData {
+	for cacheName, stats := range cacheStats.Caches {
+		cache, ok := crConfig.ContentServers[cacheName]
+		if !ok {
+			continue
+		}
+		if !strings.HasPrefix(string(cache.Type), string(tc.CacheTypeEdge)) {
+			continue
+		}
+		if len(stats.KBPS) < 1 || len(stats.MaxKBPS) < 1 {
+			continue
+		}
+		if cache.Status == "REPORTED" || cache.Status == "ONLINE" {
+			if crStates.Caches[cacheName].IsAvailable {
+				cap.Available += float64(stats.KBPS[0].Value)
+			} else {
+				cap.Unavailable += float64(stats.KBPS[0].Value)
+			}
+		} else if cache.Status == "ADMIN_DOWN" {
+			cap.Maintenance += float64(stats.KBPS[0].Value)
+		} else {
+			continue // don't add capacity for OFFLINE or other statuses
+		}
+		cap.Capacity += float64(stats.MaxKBPS[0].Value) - thresholds[cache.Profile]
+	}
+	return cap
+}
+
+func getEdgeProfileHealthThresholdBandwidth(tx *sql.Tx) (map[string]float64, error) {
+	rows, err := tx.Query(`
+SELECT pr.name as profile, pa.name, pa.config_file, pa.value
+FROM parameter as pa
+JOIN profile_parameter as pp ON pp.parameter = pa.id
+JOIN profile as pr ON pp.profile = pr.id
+JOIN server as s ON s.profile = pr.id
+JOIN cdn as c ON c.id = s.cdn_id
+JOIN type as t ON s.type = t.id
+WHERE t.name LIKE 'EDGE%'
+AND pa.config_file = 'rascal-config.txt'
+AND pa.name = 'health.threshold.availableBandwidthInKbps'
+`)
+	if err != nil {
+		return nil, errors.New("querying thresholds: " + err.Error())
+	}
+	defer rows.Close()
+	profileThresholds := map[string]float64{}
+	for rows.Next() {
+		profile := ""
+		threshStr := ""
+		if err := rows.Scan(&profile, &threshStr); err != nil {
+			return nil, errors.New("scanning thresholds: " + err.Error())
+		}
+		threshStr = strings.TrimPrefix(threshStr, ">")
+		thresh, err := strconv.ParseFloat(threshStr, 64)
+		if err != nil {
+			return nil, errors.New("profile '" + profile + "' health.threshold.availableBandwidthInKbps is not a number")
+		}
+		profileThresholds[profile] = thresh
+	}
+	return profileThresholds, nil
+}
+
+func getCRStates(monitorFQDN string, client *http.Client) (CRStates, error) {
+	path := `/publish/CrStates`
+	resp, err := client.Get("http://" + monitorFQDN + path)
+	if err != nil {
+		return CRStates{}, errors.New("getting CRStates from Monitor '" + monitorFQDN + "': " + err.Error())
+	}
+	defer resp.Body.Close()
+
+	crs := CRStates{}
+	if err := json.NewDecoder(resp.Body).Decode(&crs); err != nil {
+		return CRStates{}, errors.New("decoding CRStates from monitor '" + monitorFQDN + "': " + err.Error())
+	}
+	return crs, nil
+}
+
+func getCRConfig(monitorFQDN string, client *http.Client) (CRConfig, error) {
+	path := `/publish/CrConfig`
+	resp, err := client.Get("http://" + monitorFQDN + path)
+	if err != nil {
+		return CRConfig{}, errors.New("getting CRConfig from Monitor '" + monitorFQDN + "': " + err.Error())
+	}
+	defer resp.Body.Close()
+	crs := CRConfig{}
+	if err := json.NewDecoder(resp.Body).Decode(&crs); err != nil {
+		return CRConfig{}, errors.New("decoding CRConfig from monitor '" + monitorFQDN + "': " + err.Error())
+	}
+	return crs, nil
+}
+
+// CacheStats contains the Monitor CacheStats needed by Cachedata. It is NOT the full object served by the Monitor, but only the data required by the caches stats endpoint.
+type CacheStats struct {
+	Caches map[tc.CacheName]CacheStat `json:"caches"`
+}
+
+type CacheStat struct {
+	KBPS    []CacheStatData `json:"kbps"`
+	MaxKBPS []CacheStatData `json:"maxKbps"`
+}
+
+type CacheStatData struct {
+	Value float64 `json:"value,string"`
+}
+
+// getCacheStats gets the cache stats from the given monitor. It takes stats, a slice of stat names; and cacheStats, an object to deserialize stats into. The cacheStats type must be of the form struct {caches map[tc.CacheName]struct{statName []struct{value float64}}} with the desired stats, with appropriate member names or tags.
+func getCacheStats(monitorFQDN string, client *http.Client, stats []string, cacheStats interface{}) error {
+	path := `/publish/CacheStats`
 
 Review comment:
   am i wrong or should this just be `/CacheStats`?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services