You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2020/09/29 17:57:36 UTC
[trafficcontrol] branch master updated: Fix CacheStats api to allow
upgrading (#5031)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new e8bb3e9 Fix CacheStats api to allow upgrading (#5031)
e8bb3e9 is described below
commit e8bb3e90ab67faf4cd3841aacd3dcec6c0d371e6
Author: Steve Hamrick <sh...@users.noreply.github.com>
AuthorDate: Tue Sep 29 11:57:22 2020 -0600
Fix CacheStats api to allow upgrading (#5031)
* Fix CacheStats api to allow upgrading
* Rename and add comments
* Rename unneeded function
* Fix cache/stats with old TMs
* Use proper types in TS
* Dont allow identicle regex
* Code review changes
* Code review changes
* Code review changes
* Missed a spot, extracting constants
* Fix casting
* Forgot some files
---
lib/go-tc/invalidationjobs.go | 105 +++++++++++++---
lib/go-tc/invalidationjobs_test.go | 22 ++++
lib/go-tc/traffic_monitor.go | 135 ++++++++++++++++++++-
lib/go-tc/traffic_monitor_test.go | 131 ++++++++++++++++++++
lib/go-util/num.go | 7 ++
lib/go-util/num_test.go | 11 ++
traffic_monitor/cache/cache.go | 26 +---
traffic_monitor/cache/data.go | 52 +-------
traffic_monitor/datareq/cachestat.go | 22 +++-
traffic_monitor/datareq/datareq.go | 5 +-
traffic_monitor/datareq/peerstate.go | 5 +-
traffic_monitor/datareq/statsummary.go | 6 +-
traffic_monitor/dsdata/stat.go | 2 +-
traffic_monitor/srvhttp/srvhttp.go | 11 +-
traffic_monitor/static/index.html | 2 +-
traffic_monitor/threadsafe/resultstathistory.go | 104 +++++++++++-----
.../threadsafe/resultstathistory_test.go | 45 ++++++-
traffic_ops/testing/api/v3/jobs_test.go | 36 ++++++
.../traffic_ops_golang/cachesstats/cachesstats.go | 58 +++------
traffic_ops/traffic_ops_golang/cdn/capacity.go | 120 ++++++------------
.../traffic_ops_golang/cdn/capacity_test.go | 65 +++-------
.../traffic_ops_golang/deliveryservice/capacity.go | 35 +++---
.../util/monitorhlp/monitorhlp.go | 52 +++++++-
traffic_stats/traffic_stats.go | 41 ++-----
24 files changed, 723 insertions(+), 375 deletions(-)
diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go
index 7d109cb..7bced64 100644
--- a/lib/go-tc/invalidationjobs.go
+++ b/lib/go-tc/invalidationjobs.go
@@ -19,19 +19,21 @@ package tc
* under the License.
*/
-import "errors"
-import "fmt"
-import "regexp"
-import "database/sql"
-import "math"
-import "strconv"
-import "strings"
-import "time"
-
-import "github.com/apache/trafficcontrol/lib/go-log"
-
-import "github.com/go-ozzo/ozzo-validation"
-import "github.com/go-ozzo/ozzo-validation/is"
+import (
+ "database/sql"
+ "errors"
+ "fmt"
+ "math"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-log"
+ "github.com/go-ozzo/ozzo-validation/is"
+
+ validation "github.com/go-ozzo/ozzo-validation"
+)
// MaxTTL is the maximum value of TTL representable as a time.Duration object, which is used
// internally by InvalidationJobInput objects to store the TTL.
@@ -238,8 +240,9 @@ func (job *InvalidationJobInput) Validate(tx *sql.Tx) error {
errs = append(errs, err.Error())
}
+ var dsID uint
if job.DeliveryService != nil {
- if _, err := job.DSID(tx); err != nil {
+ if dsID, err = job.DSID(tx); err != nil {
errs = append(errs, err.Error())
}
}
@@ -272,9 +275,83 @@ func (job *InvalidationJobInput) Validate(tx *sql.Tx) error {
if len(errs) > 0 {
return errors.New(strings.Join(errs, ", "))
}
+
+ errs = job.ValidateUniqueness(tx, dsID)
+ if len(errs) > 0 {
+ return errors.New(strings.Join(errs, ", "))
+ }
+
return nil
}
+func (job *InvalidationJobInput) ValidateUniqueness(tx *sql.Tx, dsID uint) []string {
+ errors := []string{}
+
+ const readQuery = `
+SELECT job.id,
+ keyword,
+ parameters,
+ asset_url,
+ start_time
+FROM job
+WHERE job.job_deliveryservice = $1
+`
+ rows, err := tx.Query(readQuery, dsID)
+ if err != nil {
+ errors = append(errors, fmt.Sprintf("unable to query for other invalidation jobs"))
+ return errors
+ }
+
+ defer rows.Close()
+ jobStart := job.StartTime.Time
+ for rows.Next() {
+ testJob := InvalidationJob{}
+ err = rows.Scan(&testJob.ID, &testJob.Keyword, &testJob.Parameters, &testJob.AssetURL, &testJob.StartTime)
+ if err != nil {
+ continue
+ }
+ if !strings.HasSuffix(*testJob.AssetURL, *job.Regex) {
+ continue
+ }
+ testJobTTL := testJob.GetTTL()
+ if testJobTTL == 0 {
+ continue
+ }
+ testJobStart := testJob.StartTime.Time
+ testJobEnd := testJobStart.Add(time.Hour * time.Duration(testJobTTL))
+ jobTTLHours, _ := job.TTLHours()
+ jobEnd := jobStart.Add(time.Hour * time.Duration(jobTTLHours))
+ // jobStart in testJob range
+ if (testJobStart.Before(jobStart) && jobStart.Before(testJobEnd)) ||
+ // jobEnd in testJob range
+ (testJobStart.Before(jobEnd) && jobEnd.Before(testJobEnd)) ||
+ // job range encaspulates testJob range
+ (testJobEnd.Before(jobEnd) && jobStart.Before(jobStart)) {
+ errors = append(errors, fmt.Sprintf("job already has an invalidation job that overlaps, start:%v end:%v", testJobStart, testJobEnd))
+ }
+ }
+
+ return errors
+}
+
+// GetTTL will parse job.Parameters to find TTL, returns an int representing number of hours. Returns 0
+// in case of issue (0 is an invalid TTL)
+func (job *InvalidationJob) GetTTL() uint {
+ if job.Parameters == nil {
+ return 0
+ }
+ ttl := strings.Split(*job.Parameters, ":")
+ if len(ttl) != 2 {
+ return 0
+ }
+
+ hours, err := strconv.Atoi(ttl[1][:len(ttl[1])-1])
+ if err != nil {
+ return 0
+ }
+ return uint(hours)
+}
+
// Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined.
//
// This returns an error describing any and all problematic fields encountered during validation.
diff --git a/lib/go-tc/invalidationjobs_test.go b/lib/go-tc/invalidationjobs_test.go
index 0c8d3cf..dba5465 100644
--- a/lib/go-tc/invalidationjobs_test.go
+++ b/lib/go-tc/invalidationjobs_test.go
@@ -21,10 +21,32 @@ package tc
import (
"fmt"
+ "testing"
"github.com/apache/trafficcontrol/lib/go-util"
)
+func TestInvalidationJobGetTTL(t *testing.T) {
+ job := InvalidationJob{
+ Parameters: nil,
+ }
+ ttl := job.GetTTL()
+ if ttl != 0 {
+ t.Error("expected 0 when no parameters")
+ }
+ job.Parameters = util.StrPtr("TTL:24h,x:asdf")
+ ttl = job.GetTTL()
+ if ttl != 0 {
+ t.Error("expected 0 when invalid parameters")
+ }
+
+ job.Parameters = util.StrPtr("TTL:24h")
+ ttl = job.GetTTL()
+ if ttl != 24 {
+ t.Errorf("expected ttl to be 24, got %v", ttl)
+ }
+}
+
func ExampleInvalidationJobInput_TTLHours_duration() {
j := InvalidationJobInput{nil, nil, nil, util.InterfacePtr("121m"), nil, nil}
ttl, e := j.TTLHours()
diff --git a/lib/go-tc/traffic_monitor.go b/lib/go-tc/traffic_monitor.go
index a78c809..5e1bd8a 100644
--- a/lib/go-tc/traffic_monitor.go
+++ b/lib/go-tc/traffic_monitor.go
@@ -25,11 +25,19 @@ import (
"fmt"
"strconv"
"strings"
+ "time"
+
+ jsoniter "github.com/json-iterator/go"
)
-// ThresholdPrefix is the prefix of all Names of Parameters used to define
-// monitoring thresholds.
-const ThresholdPrefix = "health.threshold."
+const (
+ // ThresholdPrefix is the prefix of all Names of Parameters used to define
+ // monitoring thresholds.
+ ThresholdPrefix = "health.threshold."
+ StatNameKBPS = "kbps"
+ StatNameMaxKBPS = "maxKbps"
+ StatNameBandwidth = "bandwidth"
+)
// TMConfigResponse is the response to requests made to the
// cdns/{{Name}}/configs/monitoring endpoint of the Traffic Ops API.
@@ -151,6 +159,127 @@ type TrafficMonitorConfigMap struct {
Profile map[string]TMProfile
}
+func (s *Stats) ToLegacy(monitorConfig TrafficMonitorConfigMap) ([]string, LegacyStats) {
+ legacyStats := LegacyStats{
+ CommonAPIData: s.CommonAPIData,
+ Caches: make(map[CacheName]map[string][]ResultStatVal, len(s.Caches)),
+ }
+ skippedCaches := []string{}
+
+ for cacheName, cache := range s.Caches {
+ ts, ok := monitorConfig.TrafficServer[cacheName]
+ if !ok {
+ skippedCaches = append(skippedCaches, "Cache "+cacheName+" does not exist in the "+
+ "TrafficMonitorConfigMap")
+ continue
+ }
+ legacyInterface, err := InterfaceInfoToLegacyInterfaces(ts.Interfaces)
+ if err != nil {
+ skippedCaches = append(skippedCaches, "Cache "+cacheName+": unable to convert to legacy "+
+ "interfaces: "+err.Error())
+ continue
+ }
+ if legacyInterface.InterfaceName == nil {
+ skippedCaches = append(skippedCaches, "Cache "+cacheName+": computed legacy interface "+
+ "does not have a name")
+ continue
+ }
+ monitorInterfaceStats, ok := cache.Interfaces[*legacyInterface.InterfaceName]
+ if !ok {
+ skippedCaches = append(skippedCaches, "Cache "+cacheName+" does not contain interface "+
+ *legacyInterface.InterfaceName)
+ continue
+ }
+ length := len(monitorInterfaceStats) + len(cache.Stats)
+ legacyStats.Caches[CacheName(cacheName)] = make(map[string][]ResultStatVal, length)
+ for statName, stat := range cache.Stats {
+ legacyStats.Caches[CacheName(cacheName)][statName] = stat
+ }
+ for statName, stat := range monitorInterfaceStats {
+ legacyStats.Caches[CacheName(cacheName)][statName] = stat
+ }
+ }
+
+ return skippedCaches, legacyStats
+}
+
+// ServerStats is a representation of cache server statistics as present in the
+// TM API.
+type ServerStats struct {
+ // Interfaces contains statistics specific to each monitored interface
+ // of the cache server.
+ Interfaces map[string]map[string][]ResultStatVal `json:"interfaces"`
+ // Stats contains statistics regarding the cache server in general.
+ Stats map[string][]ResultStatVal `json:"stats"`
+}
+
+// Stats is designed for returning via the API. It contains result history
+// for each cache, as well as common API data.
+type Stats struct {
+ CommonAPIData
+ // Caches is a map of cache server hostnames to groupings of statistics
+ // regarding each cache server and all of its separate network interfaces.
+ Caches map[string]ServerStats `json:"caches"`
+}
+
+type LegacyStats struct {
+ CommonAPIData
+ Caches map[CacheName]map[string][]ResultStatVal `json:"caches"`
+}
+
+// CommonAPIData contains generic data common to most endpoints.
+type CommonAPIData struct {
+ QueryParams string `json:"pp"`
+ DateStr string `json:"date"`
+}
+
+// ResultStatVal is the value of an individual stat returned from a poll.
+// JSON values are all strings, for the TM1.0 /publish/CacheStats API.
+type ResultStatVal struct {
+ // Span is the number of polls this stat has been the same. For example,
+ // if History is set to 100, and the last 50 polls had the same value for
+ // this stat (but none of the previous 50 were the same), this stat's map
+ // value slice will actually contain 51 entries, and the first entry will
+ // have the value, the time of the last poll, and a Span of 50.
+ // Assuming the poll time is every 8 seconds, users will then know, looking
+ // at the Span, that the value was unchanged for the last 50*8=400 seconds.
+ Span uint64 `json:"span"`
+ // Time is the time this stat was returned.
+ Time time.Time `json:"time"`
+ Val interface{} `json:"value"`
+}
+
+func (t *ResultStatVal) MarshalJSON() ([]byte, error) {
+ v := struct {
+ Val string `json:"value"`
+ Time int64 `json:"time"`
+ Span uint64 `json:"span"`
+ }{
+ Val: fmt.Sprintf("%v", t.Val),
+ Time: t.Time.UnixNano() / 1000000, // ms since the epoch
+ Span: t.Span,
+ }
+ json := jsoniter.ConfigFastest // TODO make configurable
+ return json.Marshal(&v)
+}
+
+func (t *ResultStatVal) UnmarshalJSON(data []byte) error {
+ v := struct {
+ Val string `json:"value"`
+ Time int64 `json:"time"`
+ Span uint64 `json:"span"`
+ }{}
+ json := jsoniter.ConfigFastest // TODO make configurable
+ err := json.Unmarshal(data, &v)
+ if err != nil {
+ return err
+ }
+ t.Time = time.Unix(0, v.Time*1000000)
+ t.Val = v.Val
+ t.Span = v.Span
+ return nil
+}
+
// Valid returns a non-nil error if the configuration map is invalid.
//
// A configuration map is considered invalid if:
diff --git a/lib/go-tc/traffic_monitor_test.go b/lib/go-tc/traffic_monitor_test.go
index 3dc34e1..bf6d0f4 100644
--- a/lib/go-tc/traffic_monitor_test.go
+++ b/lib/go-tc/traffic_monitor_test.go
@@ -23,8 +23,139 @@ import (
"encoding/json"
"fmt"
"testing"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-util"
)
+func makeFakeStats(text string) map[string][]ResultStatVal {
+ return map[string][]ResultStatVal{
+ text + "stat1": {
+ {
+ Span: 50,
+ Time: time.Now(),
+ Val: 50,
+ },
+ },
+ text + "stat2": {
+ {
+ Span: 50,
+ Time: time.Now(),
+ Val: 50,
+ },
+ {
+ Span: 50,
+ Time: time.Now(),
+ Val: 50,
+ },
+ },
+ text + "stat3": {
+ {
+ Span: 50,
+ Time: time.Now(),
+ Val: 50,
+ },
+ {
+ Span: 50,
+ Time: time.Now(),
+ Val: 50,
+ },
+ {
+ Span: 50,
+ Time: time.Now(),
+ Val: 50,
+ },
+ },
+ }
+}
+
+func makeFakeInterfaces() map[string]map[string][]ResultStatVal {
+ return map[string]map[string][]ResultStatVal{
+ "interface1": makeFakeStats("interf"),
+ "interface2": makeFakeStats("interf"),
+ "interface3": makeFakeStats("interf"),
+ }
+}
+func TestLegacyStatsConversion(t *testing.T) {
+ stats := Stats{
+ CommonAPIData: CommonAPIData{},
+ Caches: make(map[string]ServerStats),
+ }
+ config := TrafficMonitorConfigMap{
+ TrafficServer: make(map[string]TrafficServer),
+ }
+ for _, cacheName := range []string{"cache1", "cache2", "cache3"} {
+ stats.Caches[cacheName] = ServerStats{
+ Interfaces: makeFakeInterfaces(),
+ Stats: makeFakeStats(""),
+ }
+ interfaces := []ServerInterfaceInfo{}
+ for name, _ := range stats.Caches[cacheName].Interfaces {
+ interfaces = append(interfaces, ServerInterfaceInfo{
+ IPAddresses: []ServerIPAddress{
+ {
+ Address: "192.168.0.8",
+ Gateway: util.StrPtr("192.168.0.1"),
+ ServiceAddress: true,
+ },
+ },
+ MaxBandwidth: util.Uint64Ptr(1500),
+ Monitor: false,
+ MTU: util.UInt64Ptr(1500),
+ Name: name,
+ })
+ }
+ interfaces[0].Monitor = true
+ config.TrafficServer[cacheName] = TrafficServer{
+ Interfaces: interfaces,
+ }
+ }
+
+ issues, legacyStats := stats.ToLegacy(config)
+
+ if len(issues) != 0 {
+ t.Error("expect no issues")
+ }
+
+ if legacyStats.CommonAPIData != stats.CommonAPIData {
+ t.Error("expected CommonAPIData to be the same")
+ }
+
+ if len(legacyStats.Caches) != len(stats.Caches) {
+ t.Errorf("expected %v caches, got %v", len(stats.Caches), len(legacyStats.Caches))
+ }
+
+ for cacheName, legacyCache := range legacyStats.Caches {
+ cache, ok := stats.Caches[string(cacheName)]
+ if !ok {
+ t.Errorf("new interface %v found in upgraded stats, but not in legacy stats", cacheName)
+ }
+ interf := cache.Interfaces["interface1"]
+ if len(interf)+len(cache.Stats) != len(legacyCache) {
+ t.Errorf("expected %v stats, got %v", len(interf)+len(cache.Stats), len(legacyCache))
+ }
+ }
+}
+
+func TestLegacyStatsNilConversion(t *testing.T) {
+ stats := Stats{
+ CommonAPIData: CommonAPIData{},
+ Caches: nil,
+ }
+ config := TrafficMonitorConfigMap{
+ TrafficServer: nil,
+ }
+ issues, legacyStats := stats.ToLegacy(config)
+
+ if legacyStats.CommonAPIData != stats.CommonAPIData {
+ t.Error("expected CommonAPIData to be the same")
+ }
+
+ if len(issues) != 0 {
+ t.Error("expect no issues")
+ }
+}
+
func TestLegacyMonitorConfigValid(t *testing.T) {
mc := (*LegacyTrafficMonitorConfigMap)(nil)
if LegacyMonitorConfigValid(mc) == nil {
diff --git a/lib/go-util/num.go b/lib/go-util/num.go
index 7f336e0..533e03c 100644
--- a/lib/go-util/num.go
+++ b/lib/go-util/num.go
@@ -21,6 +21,7 @@ package util
import (
"crypto/sha512"
+ "fmt"
"encoding/binary"
"encoding/json"
@@ -60,6 +61,12 @@ func ToNumeric(v interface{}) (float64, bool) {
return float64(i), true
case uint:
return float64(i), true
+ case string:
+ n, err := strconv.ParseFloat(fmt.Sprintf("%v", v), 64)
+ if err != nil {
+ return 0.0, false
+ }
+ return n, true
default:
return 0.0, false
}
diff --git a/lib/go-util/num_test.go b/lib/go-util/num_test.go
index 6537e80..83a4fb5 100644
--- a/lib/go-util/num_test.go
+++ b/lib/go-util/num_test.go
@@ -83,6 +83,17 @@ func TestJSONNameOrIDStr(t *testing.T) {
}
}
+func TestToNumeric(t *testing.T) {
+ var number interface{} = "34.59354233"
+ val, success := ToNumeric(number)
+ if !success {
+ t.Errorf("expected ToNumeric to succeed for string %v", number)
+ }
+ if val != 34.59354233 {
+ t.Errorf("expected ToNumeric to return %v, got %v", number, val)
+ }
+}
+
func TestBytesLenSplit(t *testing.T) {
{
b := []byte("abcdefghijklmnopqrstuvwxyz1234567890_-+=")
diff --git a/traffic_monitor/cache/cache.go b/traffic_monitor/cache/cache.go
index 1752828..a48f3a3 100644
--- a/traffic_monitor/cache/cache.go
+++ b/traffic_monitor/cache/cache.go
@@ -25,7 +25,6 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
- "github.com/apache/trafficcontrol/traffic_monitor/srvhttp"
"github.com/apache/trafficcontrol/traffic_monitor/todata"
)
@@ -152,25 +151,6 @@ type Stat struct {
Value interface{} `json:"value"`
}
-// ServerStats is a representation of cache server statistics as present in the
-// TM API.
-type ServerStats struct {
- // Interfaces contains statistics specific to each monitored interface
- // of the cache server.
- Interfaces map[string]map[string][]ResultStatVal `json:"interfaces"`
- // Stats contains statistics regarding the cache server in general.
- Stats map[string][]ResultStatVal `json:"stats"`
-}
-
-// Stats is designed for returning via the API. It contains result history
-// for each cache, as well as common API data.
-type Stats struct {
- srvhttp.CommonAPIData
- // Caches is a map of cache server hostnames to groupings of statistics
- // regarding each cache server and all of its separate network interfaces.
- Caches map[string]ServerStats `json:"caches"`
-}
-
// Filter filters whether stats and caches should be returned from a data set.
type Filter interface {
UseCache(tc.CacheName) bool
@@ -198,16 +178,16 @@ func ComputedStats() map[string]StatComputeFunc {
"availableBandwidthInMbps": func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
return (info.Vitals.MaxKbpsOut - info.Vitals.KbpsOut) / 1000.0
},
- "bandwidth": func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
+ tc.StatNameBandwidth: func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
return info.Vitals.KbpsOut
},
- "kbps": func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
+ tc.StatNameKBPS: func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
return info.Vitals.KbpsOut
},
"gbps": func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
return float64(info.Vitals.KbpsOut) / 1000000.0
},
- "maxKbps": func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
+ tc.StatNameMaxKBPS: func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
return info.Vitals.MaxKbpsOut
},
"loadavg": func(info ResultInfo, _ tc.TrafficServer, _ tc.TMProfile, _ tc.IsAvailable) interface{} {
diff --git a/traffic_monitor/cache/data.go b/traffic_monitor/cache/data.go
index 664501f..323023a 100644
--- a/traffic_monitor/cache/data.go
+++ b/traffic_monitor/cache/data.go
@@ -20,12 +20,9 @@ package cache
*/
import (
- "fmt"
"time"
"github.com/apache/trafficcontrol/lib/go-tc"
-
- jsoniter "github.com/json-iterator/go"
)
// AvailableStatusReported is the status string returned by caches set to
@@ -106,54 +103,7 @@ func (a ResultHistory) Copy() ResultHistory {
return b
}
-// ResultStatVal is the value of an individual stat returned from a poll.
-// JSON values are all strings, for the TM1.0 /publish/CacheStats API.
-type ResultStatVal struct {
- // Span is the number of polls this stat has been the same. For example,
- // if History is set to 100, and the last 50 polls had the same value for
- // this stat (but none of the previous 50 were the same), this stat's map
- // value slice will actually contain 51 entries, and the first entry will
- // have the value, the time of the last poll, and a Span of 50.
- // Assuming the poll time is every 8 seconds, users will then know, looking
- // at the Span, that the value was unchanged for the last 50*8=400 seconds.
- Span uint64 `json:"span"`
- // Time is the time this stat was returned.
- Time time.Time `json:"time"`
- Val interface{} `json:"value"`
-}
-
-func (t *ResultStatVal) MarshalJSON() ([]byte, error) {
- v := struct {
- Val string `json:"value"`
- Time int64 `json:"time"`
- Span uint64 `json:"span"`
- }{
- Val: fmt.Sprintf("%v", t.Val),
- Time: t.Time.UnixNano() / 1000000, // ms since the epoch
- Span: t.Span,
- }
- json := jsoniter.ConfigFastest // TODO make configurable
- return json.Marshal(&v)
-}
-
-func (t *ResultStatVal) UnmarshalJSON(data []byte) error {
- v := struct {
- Val string `json:"value"`
- Time int64 `json:"time"`
- Span uint64 `json:"span"`
- }{}
- json := jsoniter.ConfigFastest // TODO make configurable
- err := json.Unmarshal(data, &v)
- if err != nil {
- return err
- }
- t.Time = time.Unix(0, v.Time*1000000)
- t.Val = v.Val
- t.Span = v.Span
- return nil
-}
-
-func pruneStats(history []ResultStatVal, limit uint64) []ResultStatVal {
+func pruneStats(history []tc.ResultStatVal, limit uint64) []tc.ResultStatVal {
if uint64(len(history)) > limit {
history = history[:limit-1]
}
diff --git a/traffic_monitor/datareq/cachestat.go b/traffic_monitor/datareq/cachestat.go
index 16ee02e..22c02a2 100644
--- a/traffic_monitor/datareq/cachestat.go
+++ b/traffic_monitor/datareq/cachestat.go
@@ -28,12 +28,30 @@ import (
"github.com/apache/trafficcontrol/traffic_monitor/todata"
)
-func srvCacheStats(params url.Values, errorCount threadsafe.Uint, path string, toData todata.TODataThreadsafe, statResultHistory threadsafe.ResultStatHistory, statInfoHistory threadsafe.ResultInfoHistory, monitorConfig threadsafe.TrafficMonitorConfigMap, combinedStates peer.CRStatesThreadsafe, statMaxKbpses threadsafe.CacheKbpses) ([]byte, int) {
+func srvCacheStats(params url.Values, errorCount threadsafe.Uint, path string, toData todata.TODataThreadsafe,
+ statResultHistory threadsafe.ResultStatHistory, statInfoHistory threadsafe.ResultInfoHistory,
+ monitorConfig threadsafe.TrafficMonitorConfigMap, combinedStates peer.CRStatesThreadsafe,
+ statMaxKbpses threadsafe.CacheKbpses) ([]byte, int) {
filter, err := NewCacheStatFilter(path, params, toData.Get().ServerTypes)
if err != nil {
HandleErr(errorCount, path, err)
return []byte(err.Error()), http.StatusBadRequest
}
- bytes, err := threadsafe.StatsMarshall(statResultHistory, statInfoHistory.Get(), combinedStates.Get(), monitorConfig.Get(), statMaxKbpses.Get(), filter, params)
+ bytes, err := threadsafe.StatsMarshall(statResultHistory, statInfoHistory.Get(), combinedStates.Get(),
+ monitorConfig.Get(), statMaxKbpses.Get(), filter, params)
+ return WrapErrCode(errorCount, path, bytes, err)
+}
+
+func srvLegacyCacheStats(params url.Values, errorCount threadsafe.Uint, path string, toData todata.TODataThreadsafe,
+ statResultHistory threadsafe.ResultStatHistory, statInfoHistory threadsafe.ResultInfoHistory,
+ monitorConfig threadsafe.TrafficMonitorConfigMap, combinedStates peer.CRStatesThreadsafe,
+ statMaxKbpses threadsafe.CacheKbpses) ([]byte, int) {
+ filter, err := NewCacheStatFilter(path, params, toData.Get().ServerTypes)
+ if err != nil {
+ HandleErr(errorCount, path, err)
+ return []byte(err.Error()), http.StatusBadRequest
+ }
+ bytes, err := threadsafe.LegacyStatsMarshall(statResultHistory, statInfoHistory.Get(), combinedStates.Get(),
+ monitorConfig.Get(), statMaxKbpses.Get(), filter, params)
return WrapErrCode(errorCount, path, bytes, err)
}
diff --git a/traffic_monitor/datareq/datareq.go b/traffic_monitor/datareq/datareq.go
index 0bccfa6..615b661 100644
--- a/traffic_monitor/datareq/datareq.go
+++ b/traffic_monitor/datareq/datareq.go
@@ -78,9 +78,12 @@ func MakeDispatchMap(
bytes, statusCode, err := srvTRState(params, localStates, combinedStates, peerStates)
return WrapErrStatusCode(errorCount, path, bytes, statusCode, err)
}, rfc.ApplicationJSON)),
- "/publish/CacheStats": wrap(WrapParams(func(params url.Values, path string) ([]byte, int) {
+ "/publish/CacheStatsNew": wrap(WrapParams(func(params url.Values, path string) ([]byte, int) {
return srvCacheStats(params, errorCount, path, toData, statResultHistory, statInfoHistory, monitorConfig, combinedStates, statMaxKbpses)
}, rfc.ApplicationJSON)),
+ "/publish/CacheStats": wrap(WrapParams(func(params url.Values, path string) ([]byte, int) {
+ return srvLegacyCacheStats(params, errorCount, path, toData, statResultHistory, statInfoHistory, monitorConfig, combinedStates, statMaxKbpses)
+ }, rfc.ApplicationJSON)),
"/publish/DsStats": wrap(WrapParams(func(params url.Values, path string) ([]byte, int) {
return srvDSStats(params, errorCount, path, toData, dsStats)
}, rfc.ApplicationJSON)),
diff --git a/traffic_monitor/datareq/peerstate.go b/traffic_monitor/datareq/peerstate.go
index bbc0e5c..be68120 100644
--- a/traffic_monitor/datareq/peerstate.go
+++ b/traffic_monitor/datareq/peerstate.go
@@ -29,13 +29,12 @@ import (
"github.com/apache/trafficcontrol/traffic_monitor/srvhttp"
"github.com/apache/trafficcontrol/traffic_monitor/threadsafe"
"github.com/apache/trafficcontrol/traffic_monitor/todata"
-
- "github.com/json-iterator/go"
+ jsoniter "github.com/json-iterator/go"
)
// APIPeerStates contains the data to be returned for an API call to get the peer states of a Traffic Monitor. This contains common API data returned by most endpoints, and a map of peers, to caches' states.
type APIPeerStates struct {
- srvhttp.CommonAPIData
+ tc.CommonAPIData
Peers map[tc.TrafficMonitorName]map[tc.CacheName][]CacheState `json:"peers"`
}
diff --git a/traffic_monitor/datareq/statsummary.go b/traffic_monitor/datareq/statsummary.go
index d9bed95..184f935 100644
--- a/traffic_monitor/datareq/statsummary.go
+++ b/traffic_monitor/datareq/statsummary.go
@@ -78,7 +78,7 @@ type CacheStatSummary struct {
type StatSummary struct {
Caches map[string]CacheStatSummary `json:"caches"`
- srvhttp.CommonAPIData
+ tc.CommonAPIData
}
func srvStatSummary(params url.Values, errorCount threadsafe.Uint, path string, toData todata.TODataThreadsafe, statResultHistory threadsafe.ResultStatHistory) ([]byte, int) {
@@ -107,7 +107,7 @@ func createStatSummary(statResultHistory threadsafe.ResultStatHistory, filter ca
var cacheStats CacheStatSummary
ssStats := map[string]StatSummaryStat{}
- stats.Stats.Range(func(statName string, statHistory []cache.ResultStatVal) bool {
+ stats.Stats.Range(func(statName string, statHistory []tc.ResultStatVal) bool {
if !filter.UseStat(statName) || len(statHistory) == 0 {
return true
}
@@ -162,7 +162,7 @@ func createStatSummary(statResultHistory threadsafe.ResultStatHistory, filter ca
}
infStatMap := map[string]StatSummaryStat{}
- infStatHistory.Range(func(statName string, statHistory []cache.ResultStatVal) bool {
+ infStatHistory.Range(func(statName string, statHistory []tc.ResultStatVal) bool {
if !filter.UseInterfaceStat(statName) || len(statHistory) == 0 {
return true
}
diff --git a/traffic_monitor/dsdata/stat.go b/traffic_monitor/dsdata/stat.go
index 32ad5e2..dab4061 100644
--- a/traffic_monitor/dsdata/stat.go
+++ b/traffic_monitor/dsdata/stat.go
@@ -52,7 +52,7 @@ type StatOld struct {
// StatsOld is the old JSON representation of stats, from Traffic Monitor 1.0. It is designed to be serialized and returns from an API, and includes stat history for each delivery service, as well as data common to most endpoints.
type StatsOld struct {
DeliveryService map[tc.DeliveryServiceName]map[StatName][]StatOld `json:"deliveryService"`
- srvhttp.CommonAPIData
+ tc.CommonAPIData
}
// StatsReadonly is a read-only interface for delivery service Stats, designed to be passed to multiple goroutine readers.
diff --git a/traffic_monitor/srvhttp/srvhttp.go b/traffic_monitor/srvhttp/srvhttp.go
index e3dd673..c28953c 100644
--- a/traffic_monitor/srvhttp/srvhttp.go
+++ b/traffic_monitor/srvhttp/srvhttp.go
@@ -32,23 +32,18 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-rfc"
+ "github.com/apache/trafficcontrol/lib/go-tc"
"github.com/hydrogen18/stoppableListener"
)
// GetCommonAPIData calculates and returns API data common to most endpoints
-func GetCommonAPIData(params url.Values, t time.Time) CommonAPIData {
- return CommonAPIData{
+func GetCommonAPIData(params url.Values, t time.Time) tc.CommonAPIData {
+ return tc.CommonAPIData{
QueryParams: ParametersStr(params),
DateStr: DateStr(t),
}
}
-// CommonAPIData contains generic data common to most endpoints.
-type CommonAPIData struct {
- QueryParams string `json:"pp"`
- DateStr string `json:"date"`
-}
-
// Server is a re-runnable HTTP server. Server.Run() may be called repeatedly, and
// each time the previous running server will be stopped, and the server will be
// restarted with the new port address and data request channel.
diff --git a/traffic_monitor/static/index.html b/traffic_monitor/static/index.html
index 5203525..7aa8e34 100644
--- a/traffic_monitor/static/index.html
+++ b/traffic_monitor/static/index.html
@@ -38,7 +38,7 @@ under the License.
<div id="links">
<div>
<a href="/publish/EventLog">EventLog</a>
- <a href="/publish/CacheStats">CacheStats</a>
+ <a href="/publish/CacheStatsNew">CacheStats</a>
<a href="/publish/DsStats">DsStats</a>
<a href="/publish/CrStates">CrStates (as published to Traffic Routers)</a>
<a href="/publish/CrConfig">CrConfig (json)</a>
diff --git a/traffic_monitor/threadsafe/resultstathistory.go b/traffic_monitor/threadsafe/resultstathistory.go
index 521a827..0158bfc 100644
--- a/traffic_monitor/threadsafe/resultstathistory.go
+++ b/traffic_monitor/threadsafe/resultstathistory.go
@@ -129,9 +129,9 @@ type interfaceStat struct {
// a single stat for a single network interface to its historical values and do
// the appropriate appending and management of the history to ensure it never
// exceeds `limit`.
-func compareAndAppendStatForInterface(history []cache.ResultStatVal, errs strings.Builder, limit uint64, stat interfaceStat) []cache.ResultStatVal {
+func compareAndAppendStatForInterface(history []tc.ResultStatVal, errs strings.Builder, limit uint64, stat interfaceStat) []tc.ResultStatVal {
if history == nil {
- history = make([]cache.ResultStatVal, 0, limit)
+ history = make([]tc.ResultStatVal, 0, limit)
}
ok, err := newStatEqual(history, stat.Stat)
@@ -149,13 +149,13 @@ func compareAndAppendStatForInterface(history []cache.ResultStatVal, errs string
if uint64(len(history)) > limit {
history = history[:limit]
} else if uint64(len(history)) < limit {
- history = append(history, cache.ResultStatVal{})
+ history = append(history, tc.ResultStatVal{})
}
for i := len(history) - 1; i >= 1; i-- {
history[i] = history[i-1]
}
- history[0] = cache.ResultStatVal{
+ history[0] = tc.ResultStatVal{
Val: stat.Stat,
Time: stat.Time,
Span: 1,
@@ -180,7 +180,7 @@ func (a ResultStatHistory) Add(r cache.Result, limit uint64) error {
for statName, statVal := range r.Miscellaneous {
statHistory := cacheHistory.Stats.Load(statName)
if statHistory == nil {
- statHistory = make([]cache.ResultStatVal, 0, limit)
+ statHistory = make([]tc.ResultStatVal, 0, limit)
}
ok, err := newStatEqual(statHistory, statVal)
@@ -201,13 +201,13 @@ func (a ResultStatHistory) Add(r cache.Result, limit uint64) error {
if uint64(len(statHistory)) > limit {
statHistory = statHistory[:limit]
} else if uint64(len(statHistory)) < limit {
- statHistory = append(statHistory, cache.ResultStatVal{})
+ statHistory = append(statHistory, tc.ResultStatVal{})
}
for i := len(statHistory) - 1; i >= 1; i-- {
statHistory[i] = statHistory[i-1]
}
- statHistory[0] = cache.ResultStatVal{
+ statHistory[0] = tc.ResultStatVal{
Val: statVal,
Time: r.Time,
Span: 1,
@@ -271,19 +271,19 @@ func NewResultStatValHistory() ResultStatValHistory { return ResultStatValHistor
// Load returns the []ResultStatVal for the given stat. If the given stat does
// not exist, nil is returned.
-func (h ResultStatValHistory) Load(stat string) []cache.ResultStatVal {
+func (h ResultStatValHistory) Load(stat string) []tc.ResultStatVal {
i, ok := h.Map.Load(stat)
if !ok {
return nil
}
- return i.([]cache.ResultStatVal)
+ return i.([]tc.ResultStatVal)
}
// Range behaves like sync.Map.Range. It calls f for every value in the map; if
// f returns false, the iteration is stopped.
-func (h ResultStatValHistory) Range(f func(stat string, val []cache.ResultStatVal) bool) {
+func (h ResultStatValHistory) Range(f func(stat string, val []tc.ResultStatVal) bool) {
h.Map.Range(func(k, v interface{}) bool {
- return f(k.(string), v.([]cache.ResultStatVal))
+ return f(k.(string), v.([]tc.ResultStatVal))
})
}
@@ -293,7 +293,7 @@ func (h ResultStatValHistory) Range(f func(stat string, val []cache.ResultStatVa
// writer could Store() underneath it, and the first writer would then Store()
// having lost values. To safely use ResultStatValHistory with multiple writers,
// a CompareAndSwap method would have to be added.
-func (h ResultStatValHistory) Store(stat string, vals []cache.ResultStatVal) {
+func (h ResultStatValHistory) Store(stat string, vals []tc.ResultStatVal) {
h.Map.Store(stat, vals)
}
@@ -322,7 +322,7 @@ func NewCacheStatHistory() CacheStatHistory {
// history. If len(history)==0, this returns false without error. If the given
// stat is not a JSON primitive (string, number, bool), this returns an error.
// We explicitly refuse to compare arrays and objects, for performance.
-func newStatEqual(history []cache.ResultStatVal, stat interface{}) (bool, error) {
+func newStatEqual(history []tc.ResultStatVal, stat interface{}) (bool, error) {
if len(history) == 0 {
return false, nil // if there's no history, it's "not equal", i.e. store this new history
}
@@ -347,13 +347,7 @@ func newStatEqual(history []cache.ResultStatVal, stat interface{}) (bool, error)
return stat == history[0].Val, nil
}
-// StatsMarshall encodes the stats in JSON, encoding up to historyCount of each
-// stat. If statsToUse is empty, all stats are encoded; otherwise, only the
-// given stats are encoded. If `wildcard` is true, stats which contain the text
-// in each statsToUse are returned, instead of exact stat names. If cacheType is
-// not CacheTypeInvalid, only stats for the given type are returned. If hosts is
-// not empty, only the given hosts are returned.
-func StatsMarshall(
+func generateStats(
statResultHistory ResultStatHistory,
statInfo cache.ResultInfoHistory,
combinedStates tc.CRStates,
@@ -361,10 +355,10 @@ func StatsMarshall(
statMaxKbpses cache.Kbpses,
filter cache.Filter,
params url.Values,
-) ([]byte, error) {
- stats := cache.Stats{
+) tc.Stats {
+ stats := tc.Stats{
CommonAPIData: srvhttp.GetCommonAPIData(params, time.Now()),
- Caches: map[string]cache.ServerStats{},
+ Caches: map[string]tc.ServerStats{},
}
computedStats := cache.ComputedStats()
@@ -380,13 +374,13 @@ func StatsMarshall(
cacheStatResultHistory := statResultHistory.LoadOrStore(cacheId)
if _, ok := stats.Caches[cacheId]; !ok {
- stats.Caches[cacheId] = cache.ServerStats{
- Interfaces: make(map[string]map[string][]cache.ResultStatVal),
- Stats: make(map[string][]cache.ResultStatVal),
+ stats.Caches[cacheId] = tc.ServerStats{
+ Interfaces: make(map[string]map[string][]tc.ResultStatVal),
+ Stats: make(map[string][]tc.ResultStatVal),
}
}
- cacheStatResultHistory.Stats.Range(func(stat string, vals []cache.ResultStatVal) bool {
+ cacheStatResultHistory.Stats.Range(func(stat string, vals []tc.ResultStatVal) bool {
stat = "ats." + stat // legacy reasons
if !filter.UseStat(stat) {
return true
@@ -398,7 +392,7 @@ func StatsMarshall(
break
}
if _, ok := stats.Caches[cacheId].Stats[stat]; !ok {
- stats.Caches[cacheId].Stats[stat] = []cache.ResultStatVal{val}
+ stats.Caches[cacheId].Stats[stat] = []tc.ResultStatVal{val}
} else {
stats.Caches[cacheId].Stats[stat] = append(stats.Caches[cacheId].Stats[stat], val)
}
@@ -409,7 +403,7 @@ func StatsMarshall(
})
for interfaceName, interfaceHistory := range cacheStatResultHistory.Interfaces {
- interfaceHistory.Range(func(stat string, vals []cache.ResultStatVal) bool {
+ interfaceHistory.Range(func(stat string, vals []tc.ResultStatVal) bool {
if !filter.UseInterfaceStat(stat) {
return true
}
@@ -420,7 +414,7 @@ func StatsMarshall(
break
}
if _, ok := stats.Caches[cacheId].Interfaces[interfaceName]; !ok {
- stats.Caches[cacheId].Interfaces[interfaceName] = map[string][]cache.ResultStatVal{}
+ stats.Caches[cacheId].Interfaces[interfaceName] = map[string][]tc.ResultStatVal{}
}
stats.Caches[cacheId].Interfaces[interfaceName][stat] = append(stats.Caches[cacheId].Interfaces[interfaceName][stat], val)
historyCount += val.Span
@@ -450,7 +444,7 @@ func StatsMarshall(
if !filter.UseStat(stat) {
continue
}
- rv := cache.ResultStatVal{
+ rv := tc.ResultStatVal{
Span: 1,
Time: t,
Val: statValF(resultInfo, serverInfo, serverProfile, combinedStatesCache),
@@ -460,11 +454,57 @@ func StatsMarshall(
}
}
+ return stats
+}
+
+// StatsMarshall encodes the stats in JSON, encoding up to historyCount of each
+// stat. If statsToUse is empty, all stats are encoded; otherwise, only the
+// given stats are encoded. If `wildcard` is true, stats which contain the text
+// in each statsToUse are returned, instead of exact stat names. If cacheType is
+// not CacheTypeInvalid, only stats for the given type are returned. If hosts is
+// not empty, only the given hosts are returned.
+func StatsMarshall(
+ statResultHistory ResultStatHistory,
+ statInfo cache.ResultInfoHistory,
+ combinedStates tc.CRStates,
+ monitorConfig tc.TrafficMonitorConfigMap,
+ statMaxKbpses cache.Kbpses,
+ filter cache.Filter,
+ params url.Values,
+) ([]byte, error) {
+ stats := generateStats(statResultHistory, statInfo, combinedStates, monitorConfig, statMaxKbpses, filter, params)
+
json := jsoniter.ConfigFastest // TODO make configurable
return json.Marshal(stats)
}
-func pruneStats(history []cache.ResultStatVal, limit uint64) []cache.ResultStatVal {
+// LegacyStatsMarshall encodes the stats in JSON, encoding up to historyCount of each
+// stat. If statsToUse is empty, all stats are encoded; otherwise, only the
+// given stats are encoded. If `wildcard` is true, stats which contain the text
+// in each statsToUse are returned, instead of exact stat names. If cacheType is
+// not CacheTypeInvalid, only stats for the given type are returned. If hosts is
+// not empty, only the given hosts are returned.
+func LegacyStatsMarshall(
+ statResultHistory ResultStatHistory,
+ statInfo cache.ResultInfoHistory,
+ combinedStates tc.CRStates,
+ monitorConfig tc.TrafficMonitorConfigMap,
+ statMaxKbpses cache.Kbpses,
+ filter cache.Filter,
+ params url.Values,
+) ([]byte, error) {
+
+ stats := generateStats(statResultHistory, statInfo, combinedStates, monitorConfig, statMaxKbpses, filter, params)
+ skippedCaches, legacyStats := stats.ToLegacy(monitorConfig)
+ if len(skippedCaches) > 0 {
+ log.Errorln(strings.Join(skippedCaches, "\n"))
+ }
+
+ json := jsoniter.ConfigFastest // TODO make configurable
+ return json.Marshal(legacyStats)
+}
+
+func pruneStats(history []tc.ResultStatVal, limit uint64) []tc.ResultStatVal {
if uint64(len(history)) > limit {
history = history[:limit-1]
}
diff --git a/traffic_monitor/threadsafe/resultstathistory_test.go b/traffic_monitor/threadsafe/resultstathistory_test.go
index 38fd9d1..a39467d 100644
--- a/traffic_monitor/threadsafe/resultstathistory_test.go
+++ b/traffic_monitor/threadsafe/resultstathistory_test.go
@@ -49,7 +49,7 @@ func randResultStatValHistory() ResultStatValHistory {
numSlice := 5
for i := 0; i < num; i++ {
cacheName := randStr()
- vals := []cache.ResultStatVal{}
+ vals := []tc.ResultStatVal{}
for j := 0; j < numSlice; j++ {
vals = append(vals, randResultStatVal())
}
@@ -58,8 +58,8 @@ func randResultStatValHistory() ResultStatValHistory {
return a
}
-func randResultStatVal() cache.ResultStatVal {
- return cache.ResultStatVal{
+func randResultStatVal() tc.ResultStatVal {
+ return tc.ResultStatVal{
Val: uint64(rand.Int63()),
Time: time.Now(),
Span: uint64(rand.Int63()),
@@ -134,6 +134,39 @@ func (DummyFilterNever) UseCache(tc.CacheName) bool {
func (DummyFilterNever) WithinStatHistoryMax(uint64) bool {
return false
}
+func TestLegacyStatsMarshall(t *testing.T) {
+ statHist := randResultStatHistory()
+ infHist := randResultInfoHistory()
+ filter := DummyFilterNever{}
+ params := url.Values{}
+ beforeStatsMarshall := time.Now()
+ bytes, err := LegacyStatsMarshall(statHist, infHist, tc.CRStates{}, tc.TrafficMonitorConfigMap{}, cache.Kbpses{}, filter, params)
+ afterStatsMarshall := time.Now()
+ if err != nil {
+ t.Fatalf("StatsMarshall return expected nil err, actual err: %v", err)
+ }
+
+ stats := tc.LegacyStats{}
+ json := jsoniter.ConfigFastest // TODO make configurable
+ if err := json.Unmarshal(bytes, &stats); err != nil {
+ t.Fatalf("unmarshalling expected nil err, actual err: %v", err)
+ }
+
+ if stats.CommonAPIData.QueryParams != "" {
+ t.Errorf(`unmarshalling stats.CommonAPIData.QueryParams expected "", actual %v`, stats.CommonAPIData.QueryParams)
+ }
+
+ statsDate, err := time.Parse(srvhttp.CommonAPIDataDateFormat, stats.CommonAPIData.DateStr)
+ if err != nil {
+ t.Errorf(`stats.CommonAPIData.DateStr expected format %v, actual %v`, srvhttp.CommonAPIDataDateFormat, stats.CommonAPIData.DateStr)
+ }
+ if beforeStatsMarshall.Truncate(time.Second).After(statsDate) || statsDate.Truncate(time.Second).After(afterStatsMarshall.Truncate(time.Second)) { // round to second, because CommonAPIDataDateFormat is second-precision
+ t.Errorf(`unmarshalling stats.CommonAPIData.DateStr expected between %v and %v, actual %v`, beforeStatsMarshall, afterStatsMarshall, stats.CommonAPIData.DateStr)
+ }
+ if len(stats.Caches) > 0 {
+ t.Errorf(`unmarshalling stats.Caches expected empty, actual %+v`, stats.Caches)
+ }
+}
func TestStatsMarshall(t *testing.T) {
statHist := randResultStatHistory()
@@ -147,7 +180,7 @@ func TestStatsMarshall(t *testing.T) {
t.Fatalf("StatsMarshall return expected nil err, actual err: %v", err)
}
- stats := cache.Stats{}
+ stats := tc.Stats{}
json := jsoniter.ConfigFastest // TODO make configurable
if err := json.Unmarshal(bytes, &stats); err != nil {
t.Fatalf("unmarshalling expected nil err, actual err: %v", err)
@@ -175,7 +208,7 @@ func TestSystemComputedStats(t *testing.T) {
for stat, function := range stats {
if strings.HasPrefix(stat, "system.") {
computedStat := function(cache.ResultInfo{}, tc.TrafficServer{}, tc.TMProfile{}, tc.IsAvailable{})
- _, err := newStatEqual([]cache.ResultStatVal{{Val: float64(0)}}, computedStat)
+ _, err := newStatEqual([]tc.ResultStatVal{{Val: float64(0)}}, computedStat)
if err != nil {
t.Errorf("expected no errors from newStatEqual: %s", err)
}
@@ -183,7 +216,7 @@ func TestSystemComputedStats(t *testing.T) {
}
}
-func TestcompareAndAppendStatForInterface(t *testing.T) {
+func TestCompareAndAppendStatForInterface(t *testing.T) {
var errs strings.Builder
var limit uint64 = 1
stat := interfaceStat{
diff --git a/traffic_ops/testing/api/v3/jobs_test.go b/traffic_ops/testing/api/v3/jobs_test.go
index bb5d983..311051d 100644
--- a/traffic_ops/testing/api/v3/jobs_test.go
+++ b/traffic_ops/testing/api/v3/jobs_test.go
@@ -22,6 +22,8 @@ import (
"testing"
"time"
+ "github.com/apache/trafficcontrol/lib/go-util"
+
"github.com/apache/trafficcontrol/lib/go-tc"
)
@@ -33,6 +35,7 @@ func TestJobs(t *testing.T) {
GetTestJobsQueryParams(t)
GetTestJobs(t)
GetTestInvalidationJobs(t)
+ VerifyUniqueJobTest(t)
})
}
@@ -64,6 +67,39 @@ func CreateTestJobs(t *testing.T) {
}
}
+func VerifyUniqueJobTest(t *testing.T) {
+ startTime := tc.Time{
+ Time: time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC),
+ Valid: true,
+ }
+ firstJob := tc.InvalidationJobInput{
+ DeliveryService: util.InterfacePtr(testData.DeliveryServices[0].XMLID),
+ Regex: util.StrPtr(`/\.*([A-Z]0?)`),
+ TTL: util.InterfacePtr(8),
+ StartTime: &startTime,
+ }
+ _, _, err := TOSession.CreateInvalidationJob(firstJob)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ newTime := tc.Time{
+ Time: startTime.Time.Add(time.Hour),
+ Valid: true,
+ }
+ newJob := tc.InvalidationJobInput{
+ DeliveryService: firstJob.DeliveryService,
+ Regex: firstJob.Regex,
+ TTL: firstJob.TTL,
+ StartTime: &newTime,
+ }
+
+ _, _, err = TOSession.CreateInvalidationJob(newJob)
+ if err == nil {
+ t.Fatal("expected invalidation job create to fail")
+ }
+}
+
func CreateTestInvalidationJobs(t *testing.T) {
toDSes, _, err := TOSession.GetDeliveryServicesNullable()
if err != nil {
diff --git a/traffic_ops/traffic_ops_golang/cachesstats/cachesstats.go b/traffic_ops/traffic_ops_golang/cachesstats/cachesstats.go
index cba0e22..0f49031 100644
--- a/traffic_ops/traffic_ops_golang/cachesstats/cachesstats.go
+++ b/traffic_ops/traffic_ops_golang/cachesstats/cachesstats.go
@@ -21,7 +21,6 @@ package cachesstats
import (
"database/sql"
- "encoding/json"
"errors"
"net/http"
"strconv"
@@ -33,6 +32,8 @@ import (
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/monitorhlp"
)
+const ATSCurrentConnectionsStat = "ats.proxy.process.http.current_client_connections"
+
func Get(w http.ResponseWriter, r *http.Request) {
inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil)
if userErr != nil || sysErr != nil {
@@ -77,10 +78,16 @@ func getCachesStats(tx *sql.Tx) ([]CacheData, error) {
continue
}
- cacheStats, err := getCacheStats(monitorFQDN, client)
+ var cacheStats tc.Stats
+ stats := []string{ATSCurrentConnectionsStat, tc.StatNameBandwidth}
+ cacheStats, err = monitorhlp.GetCacheStats(monitorFQDN, client, stats)
if err != nil {
- errs = append(errs, errors.New("getting CacheStats for CDN '"+string(cdn)+"' monitor '"+monitorFQDN+"': "+err.Error()))
- continue
+ legacyCacheStats, err := monitorhlp.GetLegacyCacheStats(monitorFQDN, client, stats)
+ if err != nil {
+ errs = append(errs, errors.New("getting CacheStats for CDN '"+string(cdn)+"' monitor '"+monitorFQDN+"': "+err.Error()))
+ continue
+ }
+ cacheStats = monitorhlp.UpgradeLegacyStats(legacyCacheStats)
}
cacheData = addHealth(cacheData, crStates)
@@ -128,49 +135,22 @@ func addTotals(data []CacheData) []CacheData {
return data
}
-// CRStates contains the Monitor CacheStats needed by Cachedata. It is NOT the full object served by the Monitor, but only the data required by the caches stats endpoint.
-type CacheStats struct {
- Caches map[tc.CacheName]CacheStat `json:"caches"`
-}
-
-type CacheStat struct {
- BandwidthKBPS []CacheStatData `json:"bandwidth"`
- Connections []CacheStatData `json:"ats.proxy.process.http.current_client_connections"`
-}
-
-type CacheStatData struct {
- Value int64 `json:"value,string"`
-}
-
-func getCacheStats(monitorFQDN string, client *http.Client) (CacheStats, error) {
- path := `/publish/CacheStats?stats=ats.proxy.process.http.current_client_connections,bandwidth`
- resp, err := client.Get("http://" + monitorFQDN + path)
- if err != nil {
- return CacheStats{}, errors.New("getting CacheStats from Monitor '" + monitorFQDN + "': " + err.Error())
- }
- defer resp.Body.Close()
-
- cs := CacheStats{}
- if err := json.NewDecoder(resp.Body).Decode(&cs); err != nil {
- return CacheStats{}, errors.New("decoding CacheStats from monitor '" + monitorFQDN + "': " + err.Error())
- }
- return cs, nil
-}
-
-func addStats(cacheData []CacheData, stats CacheStats) []CacheData {
+func addStats(cacheData []CacheData, stats tc.Stats) []CacheData {
if stats.Caches == nil {
return cacheData // TODO warn?
}
for i, cache := range cacheData {
- stat, ok := stats.Caches[cache.HostName]
+ stat, ok := stats.Caches[string(cache.HostName)]
if !ok {
continue
}
- if len(stat.BandwidthKBPS) > 0 {
- cache.KBPS = uint64(stat.BandwidthKBPS[0].Value)
+ bandwidth, ok := stat.Stats[tc.StatNameBandwidth]
+ if ok && len(bandwidth) > 0 {
+ cache.KBPS = bandwidth[0].Val.(uint64)
}
- if len(stat.Connections) > 0 {
- cache.Connections = uint64(stat.Connections[0].Value)
+ connections, ok := stat.Stats[ATSCurrentConnectionsStat]
+ if ok && len(connections) > 0 {
+ cache.Connections = connections[0].Val.(uint64)
}
cacheData[i] = cache
}
diff --git a/traffic_ops/traffic_ops_golang/cdn/capacity.go b/traffic_ops/traffic_ops_golang/cdn/capacity.go
index 8daf490..32ce06e 100644
--- a/traffic_ops/traffic_ops_golang/cdn/capacity.go
+++ b/traffic_ops/traffic_ops_golang/cdn/capacity.go
@@ -22,7 +22,6 @@ package cdn
import (
"crypto/tls"
"database/sql"
- "encoding/json"
"errors"
"net/http"
"net/url"
@@ -30,6 +29,8 @@ import (
"strings"
"time"
+ "github.com/apache/trafficcontrol/lib/go-util"
+
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
@@ -129,7 +130,7 @@ func getCapacityData(monitors map[tc.CDNName][]string, thresholds map[string]flo
for _, monitorFQDN := range monitorFQDNs {
crStates := tc.CRStates{}
crConfig := tc.CRConfig{}
- cacheStats := CacheStats{}
+ cacheStats := tc.Stats{}
if crStates, err = monitorhlp.GetCRStates(monitorFQDN, client); err != nil {
err = errors.New("getting CRStates for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error())
log.Warnln("getCapacity failed to get CRStates from cdn '" + string(cdn) + " monitor '" + monitorFQDN + "', trying next monitor: " + err.Error())
@@ -140,10 +141,17 @@ func getCapacityData(monitors map[tc.CDNName][]string, thresholds map[string]flo
log.Warnln("getCapacity failed to get CRConfig from cdn '" + string(cdn) + " monitor '" + monitorFQDN + "', trying next monitor: " + err.Error())
continue
}
- if err := getCacheStats(monitorFQDN, client, []string{"kbps", "maxKbps"}, &cacheStats); err != nil {
+ statsToFetch := []string{tc.StatNameKBPS, tc.StatNameMaxKBPS}
+ if cacheStats, err = monitorhlp.GetCacheStats(monitorFQDN, client, statsToFetch); err != nil {
err = errors.New("getting cache stats for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error())
- log.Warnln("getCapacity failed to get CacheStats from cdn '" + string(cdn) + " monitor '" + monitorFQDN + "', trying next monitor: " + err.Error())
- continue
+ log.Warnln("getCapacity failed to get CacheStatsNew from cdn '" + string(cdn) + " monitor '" + monitorFQDN + "', trying CacheStats" + err.Error())
+ legacyCacheStats, err := monitorhlp.GetLegacyCacheStats(monitorFQDN, client, statsToFetch)
+ if err != nil {
+ err = errors.New("getting cache stats for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error())
+ log.Warnln("getCapacity failed to get CacheStats from cdn '" + string(cdn) + " monitor '" + monitorFQDN + "', trying next monitor: " + err.Error())
+ continue
+ }
+ cacheStats = monitorhlp.UpgradeLegacyStats(legacyCacheStats)
}
cap = addCapacity(cap, cacheStats, crStates, crConfig, thresholds, tx)
@@ -156,14 +164,9 @@ func getCapacityData(monitors map[tc.CDNName][]string, thresholds map[string]flo
return cap, nil
}
-func addCapacity(cap CapData, cacheStats CacheStats, crStates tc.CRStates, crConfig tc.CRConfig, thresholds map[string]float64, tx *sql.Tx) CapData {
- serviceInterfaces, err := getServiceInterfaces(tx)
- if err != nil {
- log.Errorf("couldn't get the service interfaces for servers. err: %v", err.Error())
- return cap
- }
+func addCapacity(cap CapData, cacheStats tc.Stats, crStates tc.CRStates, crConfig tc.CRConfig, thresholds map[string]float64, tx *sql.Tx) CapData {
for cacheName, stats := range cacheStats.Caches {
- cache, ok := crConfig.ContentServers[string(cacheName)]
+ cache, ok := crConfig.ContentServers[(cacheName)]
if !ok {
continue
}
@@ -174,17 +177,13 @@ func addCapacity(cap CapData, cacheStats CacheStats, crStates tc.CRStates, crCon
if !strings.HasPrefix(*cache.ServerType, string(tc.CacheTypeEdge)) {
continue
}
- if _, ok := serviceInterfaces[string(cacheName)]; !ok {
- log.Errorf("no service interface found for server with host name %v", cacheName)
- continue
- }
- kbps, maxKbps, err := getStatsFromServiceInterface(stats[tc.InterfaceName(serviceInterfaces[string(cacheName)])])
+ kbps, maxKbps, err := getStats(stats)
if err != nil {
- log.Errorf("couldn't get service interface stats for %v. err: %v", cacheName, err.Error())
+ log.Errorf("couldn't get stats for %v. err: %v", cacheName, err.Error())
continue
}
if string(*cache.ServerStatus) == string(tc.CacheStatusReported) || string(*cache.ServerStatus) == string(tc.CacheStatusOnline) {
- if crStates.Caches[cacheName].IsAvailable {
+ if crStates.Caches[tc.CacheName(cacheName)].IsAvailable {
cap.Available += kbps
} else {
cap.Unavailable += kbps
@@ -199,42 +198,28 @@ func addCapacity(cap CapData, cacheStats CacheStats, crStates tc.CRStates, crCon
return cap
}
-func getStatsFromServiceInterface(stats CacheStat) (float64, float64, error) {
- var kbps, maxKbps float64
- if len(stats.KBPS) < 1 ||
- len(stats.MaxKBPS) < 1 {
- return kbps, maxKbps, errors.New("no kbps/ maxKbps stats to return")
+func getStats(stats tc.ServerStats) (float64, float64, error) {
+ kbpsRaw, ok := stats.Stats[tc.StatNameKBPS]
+ if !ok {
+ return 0, 0, errors.New("no kbps stats")
}
- kbps = stats.KBPS[0].Value
- maxKbps = stats.MaxKBPS[0].Value
- return kbps, maxKbps, nil
-}
-
-func getServiceInterfaces(tx *sql.Tx) (map[string]string, error) {
- query := `
-SELECT s.host_name, i.interface FROM ip_address i
-JOIN server s ON s.id = i.server
-WHERE i.service_address=true
-`
- rows, err := tx.Query(query)
- if err != nil {
- log.Errorf("couldn't get service interfaces %v", err.Error())
- return nil, err
+ maxKbpsRaw, ok := stats.Stats[tc.StatNameMaxKBPS]
+ if !ok {
+ return 0, 0, errors.New("no maxKbpsR stats")
}
- defer rows.Close()
-
- resultMap := make(map[string]string)
- for rows.Next() {
- hostname := ""
- serviceinterface := ""
- err = rows.Scan(&hostname, &serviceinterface)
- if err != nil {
- log.Errorf("error unmarshalling response %v", err.Error())
- continue
- }
- resultMap[hostname] = serviceinterface
+ if len(kbpsRaw) < 1 ||
+ len(maxKbpsRaw) < 1 {
+ return 0, 0, errors.New("no kbps/maxKbps stats to return")
}
- return resultMap, nil
+ kbps, ok := util.ToNumeric(kbpsRaw[0].Val)
+ if !ok {
+ return 0, 0, errors.New("unable to convert kbps to a float")
+ }
+ maxKbps, ok := util.ToNumeric(maxKbpsRaw[0].Val)
+ if !ok {
+ return 0, 0, errors.New("unable to convert maxKbps to a float")
+ }
+ return kbps, maxKbps, nil
}
func getEdgeProfileHealthThresholdBandwidth(tx *sql.Tx) (map[string]float64, error) {
@@ -271,37 +256,6 @@ AND pa.name = 'health.threshold.availableBandwidthInKbps'
return profileThresholds, nil
}
-// CacheStats contains the Monitor CacheStats needed by Cachedata. It is NOT the full object served by the Monitor, but only the data required by the caches stats endpoint.
-type CacheStats struct {
- Caches map[tc.CacheName]map[tc.InterfaceName]CacheStat `json:"caches"`
-}
-
-type CacheStat struct {
- KBPS []CacheStatData `json:"kbps"`
- MaxKBPS []CacheStatData `json:"maxKbps"`
-}
-
-type CacheStatData struct {
- Value float64 `json:"value,string"`
-}
-
-// getCacheStats gets the cache stats from the given monitor. It takes stats, a slice of stat names; and cacheStats, an object to deserialize stats into. The cacheStats type must be of the form struct {caches map[tc.CacheName]struct{statName []struct{value float64}}} with the desired stats, with appropriate member names or tags.
-func getCacheStats(monitorFQDN string, client *http.Client, stats []string, cacheStats interface{}) error {
- path := `/publish/CacheStats`
- if len(stats) > 0 {
- path += `?stats=` + strings.Join(stats, `,`)
- }
- resp, err := client.Get("http://" + monitorFQDN + path)
- if err != nil {
- return errors.New("getting CacheStats from Monitor '" + monitorFQDN + "': " + err.Error())
- }
- defer resp.Body.Close()
- if err := json.NewDecoder(resp.Body).Decode(cacheStats); err != nil {
- return errors.New("decoding CacheStats from monitor '" + monitorFQDN + "': " + err.Error())
- }
- return nil
-}
-
// getCDNMonitors returns an FQDN, including port, of an online monitor for each CDN. If a CDN has no online monitors, that CDN will not have an entry in the map. If a CDN has multiple online monitors, an arbitrary one will be returned.
func getCDNMonitorFQDNs(tx *sql.Tx) (map[tc.CDNName][]string, error) {
rows, err := tx.Query(`
diff --git a/traffic_ops/traffic_ops_golang/cdn/capacity_test.go b/traffic_ops/traffic_ops_golang/cdn/capacity_test.go
index c1e7b32..74e56f2 100644
--- a/traffic_ops/traffic_ops_golang/cdn/capacity_test.go
+++ b/traffic_ops/traffic_ops_golang/cdn/capacity_test.go
@@ -1,9 +1,9 @@
package cdn
import (
- "github.com/jmoiron/sqlx"
- "gopkg.in/DATA-DOG/go-sqlmock.v1"
"testing"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
)
/*
@@ -25,57 +25,20 @@ import (
* under the License.
*/
-func TestGetServiceInterfaces(t *testing.T) {
- mockDB, mock, err := sqlmock.New()
- if err != nil {
- t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
- }
- defer mockDB.Close()
-
- db := sqlx.NewDb(mockDB, "sqlmock")
- defer db.Close()
- cols := []string{"host_name", "interface"}
- rows := sqlmock.NewRows(cols)
- rows = rows.AddRow(
- "host1",
- "eth1",
- )
- rows = rows.AddRow(
- "host2",
- "eth2",
- )
- mock.ExpectBegin()
- mock.ExpectQuery("SELECT").WillReturnRows(rows)
- mock.ExpectCommit()
-
- m, err := getServiceInterfaces(db.MustBegin().Tx)
- if err != nil {
- t.Errorf("Expected no error, but got %v", err.Error())
- }
- if len(m) != 2 {
- t.Errorf("Expected a result of length %v, got %v instead", 2, len(m))
- }
- if m["host1"] != "eth1" {
- t.Errorf("Expected host1 to have service interface eth1, got %v instead", m["host1"])
- }
- if m["host2"] != "eth2" {
- t.Errorf("Expected host2 to have service interface eth2, got %v instead", m["host2"])
- }
-}
-
func TestGetStatsFromServiceInterface(t *testing.T) {
- var data1 []CacheStatData
- var data2 []CacheStatData
- kbpsData := CacheStatData{Value: 24.5}
- maxKbpsData := CacheStatData{Value: 66.8}
- data1 = append(data1, kbpsData)
- data2 = append(data2, maxKbpsData)
-
- c := CacheStat{
- KBPS: data1,
- MaxKBPS: data2,
+ data1 := tc.ServerStats{
+ Interfaces: nil,
+ Stats: map[string][]tc.ResultStatVal{
+ "kbps": {
+ {Val: 24.5},
+ },
+ "maxKbps": {
+ {Val: 66.8},
+ },
+ },
}
- kbps, maxKbps, err := getStatsFromServiceInterface(c)
+
+ kbps, maxKbps, err := getStats(data1)
if err != nil {
t.Errorf("Expected no error, but got %v", err.Error())
}
diff --git a/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go b/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go
index 1805608..d4de9dd 100644
--- a/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go
+++ b/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go
@@ -29,7 +29,6 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
- tmcache "github.com/apache/trafficcontrol/traffic_monitor/cache"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant"
@@ -114,9 +113,14 @@ func getCapacity(tx *sql.Tx, ds tc.DeliveryServiceName, cdn tc.CDNName) (Capacit
if err != nil {
return CapacityResp{}, errors.New("getting CRConfig for delivery service '" + string(ds) + "' monitor '" + monitorFQDN + "': " + err.Error())
}
- cacheStats, err := monitorhlp.GetCacheStats(monitorFQDN, client, []string{"maxKbps", "kbps"})
+ statsoFetch := []string{tc.StatNameMaxKBPS, tc.StatNameKBPS}
+ cacheStats, err := monitorhlp.GetCacheStats(monitorFQDN, client, statsoFetch)
if err != nil {
- return CapacityResp{}, errors.New("getting CacheStats for delivery service '" + string(ds) + "' monitor '" + monitorFQDN + "': " + err.Error())
+ legacyCacheStats, err := monitorhlp.GetLegacyCacheStats(monitorFQDN, client, statsoFetch)
+ if err != nil {
+ return CapacityResp{}, errors.New("getting CacheStats for delivery service '" + string(ds) + "' monitor '" + monitorFQDN + "': " + err.Error())
+ }
+ cacheStats = monitorhlp.UpgradeLegacyStats(legacyCacheStats)
}
cap := addCapacity(CapData{}, ds, cacheStats, crStates, crConfig, thresholds)
if cap.Capacity == 0 {
@@ -131,21 +135,18 @@ func getCapacity(tx *sql.Tx, ds tc.DeliveryServiceName, cdn tc.CDNName) (Capacit
}, nil
}
-const StatNameKBPS = "kbps"
-const StatNameMaxKBPS = "maxKbps"
-
func addCapacity(
cap CapData,
ds tc.DeliveryServiceName,
- cacheStats tmcache.Stats,
+ cacheStats tc.Stats,
crStates tc.CRStates,
crConfig tc.CRConfig,
thresholds map[string]float64,
) CapData {
for cacheName, statsCache := range cacheStats.Caches {
- cache, ok := crConfig.ContentServers[cacheName]
+ cache, ok := crConfig.ContentServers[string(cacheName)]
if !ok {
- log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + cacheName + "' in CacheStats but not CRConfig, skipping")
+ log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' in CacheStats but not CRConfig, skipping")
continue
}
@@ -157,27 +158,27 @@ func addCapacity(
}
stat := statsCache.Stats
- if len(stat[StatNameKBPS]) < 1 || len(stat[StatNameMaxKBPS]) < 1 {
- log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + cacheName + "' CacheStats has no kbps or maxKbps, skipping")
+ if len(stat[tc.StatNameKBPS]) < 1 || len(stat[tc.StatNameMaxKBPS]) < 1 {
+ log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CacheStats has no kbps or maxKbps, skipping")
continue
}
- kbps, err := statToFloat(stat[StatNameKBPS][0].Val)
+ kbps, err := statToFloat(stat[tc.StatNameKBPS][0].Val)
if err != nil {
- log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + cacheName + "' CacheStats kbps is not a number, skipping")
+ log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CacheStats kbps is not a number, skipping")
continue
}
- maxKBPS, err := statToFloat(stat[StatNameMaxKBPS][0].Val)
+ maxKBPS, err := statToFloat(stat[tc.StatNameMaxKBPS][0].Val)
if err != nil {
- log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + cacheName + "' CacheStats maxKps is not a number, skipping")
+ log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CacheStats maxKps is not a number, skipping")
continue
}
if cache.ServerStatus == nil {
- log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + cacheName + "' CRConfig Status is nil, skipping")
+ log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CRConfig Status is nil, skipping")
continue
}
if cache.Profile == nil {
- log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + cacheName + "' CRConfig Profile is nil, skipping")
+ log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CRConfig Profile is nil, skipping")
continue
}
if tc.CacheStatus(*cache.ServerStatus) == tc.CacheStatusReported || tc.CacheStatus(*cache.ServerStatus) == tc.CacheStatusOnline {
diff --git a/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go b/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go
index 8b0edad..3c43a97 100644
--- a/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go
+++ b/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go
@@ -31,7 +31,6 @@ import (
"time"
"github.com/apache/trafficcontrol/lib/go-tc"
- tmcache "github.com/apache/trafficcontrol/traffic_monitor/cache"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers"
)
@@ -123,20 +122,61 @@ func GetCRConfig(monitorFQDN string, client *http.Client) (tc.CRConfig, error) {
return crs, nil
}
-// GetCacheStats gets the cache stats from the given monitor. The stats parameters is which stats to get; if stats is empty or nil, all stats are fetched.
-func GetCacheStats(monitorFQDN string, client *http.Client, stats []string) (tmcache.Stats, error) {
+// GetCacheStats gets the cache stats from the given monitor. The stats parameters is which stats to get;
+// if stats is empty or nil, all stats are fetched.
+func GetCacheStats(monitorFQDN string, client *http.Client, stats []string) (tc.Stats, error) {
+ path := `/publish/CacheStatsNew?hc=1`
+ if len(stats) > 0 {
+ path += `&stats=` + strings.Join(stats, `,`)
+ }
+ resp, err := client.Get("http://" + monitorFQDN + path)
+ if err != nil {
+ return tc.Stats{}, errors.New("getting CacheStatsNew from Monitor '" + monitorFQDN + "': " + err.Error())
+ }
+ defer resp.Body.Close()
+ cacheStats := tc.Stats{}
+ if err := json.NewDecoder(resp.Body).Decode(&cacheStats); err != nil {
+ return tc.Stats{}, errors.New("decoding CacheStatsNew from monitor '" + monitorFQDN + "': " + err.Error())
+ }
+ return cacheStats, nil
+}
+
+// GetLegacyCacheStats gets the pre ATCv5.0 cache stats from the given monitor. The stats parameters is which stats to
+// get; if stats is empty or nil, all stats are fetched.
+func GetLegacyCacheStats(monitorFQDN string, client *http.Client, stats []string) (tc.LegacyStats, error) {
path := `/publish/CacheStats?hc=1`
if len(stats) > 0 {
path += `&stats=` + strings.Join(stats, `,`)
}
resp, err := client.Get("http://" + monitorFQDN + path)
if err != nil {
- return tmcache.Stats{}, errors.New("getting CacheStats from Monitor '" + monitorFQDN + "': " + err.Error())
+ return tc.LegacyStats{}, errors.New("getting CacheStats from Monitor '" + monitorFQDN + "': " + err.Error())
}
defer resp.Body.Close()
- cacheStats := tmcache.Stats{}
+ cacheStats := tc.LegacyStats{}
if err := json.NewDecoder(resp.Body).Decode(&cacheStats); err != nil {
- return tmcache.Stats{}, errors.New("decoding CacheStats from monitor '" + monitorFQDN + "': " + err.Error())
+ return tc.LegacyStats{}, errors.New("decoding CacheStats from monitor '" + monitorFQDN + "': " + err.Error())
}
return cacheStats, nil
}
+
+// UpgradeLegacyStats will take LegacyStats and transform them to Stats. It assumes all stats that go in
+// Stats.Caches[cacheName] exist in Stats and not Interfaces
+func UpgradeLegacyStats(legacyStats tc.LegacyStats) tc.Stats {
+ stats := tc.Stats{
+ CommonAPIData: legacyStats.CommonAPIData,
+ Caches: make(map[string]tc.ServerStats, len(legacyStats.Caches)),
+ }
+
+ for cacheName, cache := range legacyStats.Caches {
+ stats.Caches[string(cacheName)] = tc.ServerStats{
+ Interfaces: nil,
+ Stats: make(map[string][]tc.ResultStatVal, len(cache)),
+ }
+ for statName, stat := range cache {
+ stats.Caches[string(cacheName)].Stats[statName] = stat
+ }
+ }
+
+ return stats
+}
diff --git a/traffic_stats/traffic_stats.go b/traffic_stats/traffic_stats.go
index dc15582..a4aef4e 100644
--- a/traffic_stats/traffic_stats.go
+++ b/traffic_stats/traffic_stats.go
@@ -38,7 +38,7 @@ import (
"github.com/apache/trafficcontrol/lib/go-tc"
"github.com/apache/trafficcontrol/lib/go-util"
- "github.com/apache/trafficcontrol/traffic_ops/v2-client"
+ client "github.com/apache/trafficcontrol/traffic_ops/v2-client"
log "github.com/cihub/seelog"
influx "github.com/influxdata/influxdb/client/v2"
)
@@ -639,18 +639,7 @@ func calcDsValues(rascalData []byte, cdnName string, sampleTime int64, config St
}
func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cacheMap map[string]tc.Server, config StartupConfig) error {
-
- type CacheStatsJSON struct {
- Pp string `json:"pp"`
- Date string `json:"date"`
- Caches map[string]map[string][]struct {
- Index uint64 `json:"index"`
- Time int `json:"time"`
- Value string `json:"value"`
- Span uint64 `json:"span"`
- } `json:"caches"`
- }
- var jData CacheStatsJSON
+ var jData tc.LegacyStats
err := json.Unmarshal(trafmonData, &jData)
if err != nil {
return fmt.Errorf("could not unmarshall cache stats JSON - %v", err)
@@ -666,37 +655,27 @@ func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cache
errHndlr(err, ERROR)
}
for cacheName, cacheData := range jData.Caches {
- cache := cacheMap[cacheName]
+ cache := cacheMap[string(cacheName)]
for statName, statData := range cacheData {
//Get the stat time and make sure it's greater than the time 24 hours ago. If not, skip it so influxdb doesn't throw retention policy errors.
- validTime := time.Now().AddDate(0, 0, -1).UnixNano() / 1000000
- timeStamp := int64(statData[0].Time)
- if timeStamp < validTime {
- log.Info(fmt.Sprintf("Skipping %v %v: %v is greater than 24 hours old.", cacheName, statName, timeStamp))
+ validTime := time.Now().AddDate(0, 0, -1)
+ if statData[0].Time.Before(validTime) {
+ log.Info(fmt.Sprintf("Skipping %v %v: %v is greater than 24 hours old.", cacheName, statName, statData[0].Time))
continue
}
dataKey := statName
dataKey = strings.Replace(dataKey, ".bandwidth", ".kbps", 1)
dataKey = strings.Replace(dataKey, "-", "_", -1)
- //Get the stat time and convert to epoch
- statTime := strconv.Itoa(statData[0].Time)
- msInt, err := strconv.ParseInt(statTime, 10, 64)
- if err != nil {
- errHndlr(err, ERROR)
- }
-
- newTime := time.Unix(0, msInt*int64(time.Millisecond))
//Get the stat value and convert to float
- statValue := statData[0].Value
- statFloatValue, err := strconv.ParseFloat(statValue, 64)
- if err != nil {
+ statFloatValue, ok := statData[0].Val.(float64)
+ if !ok {
statFloatValue = 0.00
}
tags := map[string]string{
"cachegroup": cache.Cachegroup,
- "hostname": cacheName,
+ "hostname": string(cacheName),
"cdn": cdnName,
"type": cache.Type,
}
@@ -708,7 +687,7 @@ func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cache
dataKey,
tags,
fields,
- newTime,
+ statData[0].Time,
)
if err != nil {
errHndlr(err, ERROR)