You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by dg...@apache.org on 2018/08/13 14:17:45 UTC
[trafficcontrol] branch master updated (7584c44 -> 01fab08)
This is an automated email from the ASF dual-hosted git repository.
dgelinas pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git.
from 7584c44 add to-ping
new 665fe10 Add TM poller plugins
new 01fab08 Add TM noop stat parser type
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lib/go-tc/traffic_monitor.go | 17 ++
traffic_monitor/cache/cache.go | 11 +-
traffic_monitor/cache/stats_type_astats.go | 9 +-
traffic_monitor/cache/stats_type_noop.go | 50 ++++
traffic_monitor/fetcher/fetcher.go | 83 ------
traffic_monitor/manager/manager.go | 18 +-
traffic_monitor/manager/monitorconfig.go | 30 +-
traffic_monitor/poller/cache.go | 241 +++++++++++++++
traffic_monitor/poller/monitorconfig.go | 119 ++++++++
traffic_monitor/poller/poller.go | 324 ---------------------
traffic_monitor/poller/poller_type_http.go | 127 ++++++++
.../crconfighist.go => poller/poller_type_noop.go} | 18 +-
traffic_monitor/poller/poller_types.go | 75 +++++
13 files changed, 673 insertions(+), 449 deletions(-)
create mode 100644 traffic_monitor/cache/stats_type_noop.go
delete mode 100644 traffic_monitor/fetcher/fetcher.go
create mode 100644 traffic_monitor/poller/cache.go
create mode 100644 traffic_monitor/poller/monitorconfig.go
delete mode 100644 traffic_monitor/poller/poller.go
create mode 100644 traffic_monitor/poller/poller_type_http.go
copy traffic_monitor/{datareq/crconfighist.go => poller/poller_type_noop.go} (75%)
create mode 100644 traffic_monitor/poller/poller_types.go
[trafficcontrol] 01/02: Add TM poller plugins
Posted by dg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dgelinas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
commit 665fe1006693456cca1739d1273244b47fe98c8c
Author: Robert Butts <ro...@apache.org>
AuthorDate: Sun Aug 12 16:04:01 2018 -0600
Add TM poller plugins
---
lib/go-tc/traffic_monitor.go | 17 ++
traffic_monitor/fetcher/fetcher.go | 83 --------
traffic_monitor/manager/manager.go | 18 +-
traffic_monitor/manager/monitorconfig.go | 30 +--
traffic_monitor/poller/cache.go | 241 +++++++++++++++++++++
traffic_monitor/poller/monitorconfig.go | 119 +++++++++++
traffic_monitor/poller/poller.go | 324 -----------------------------
traffic_monitor/poller/poller_type_http.go | 127 +++++++++++
traffic_monitor/poller/poller_type_noop.go | 34 +++
traffic_monitor/poller/poller_types.go | 75 +++++++
10 files changed, 637 insertions(+), 431 deletions(-)
diff --git a/lib/go-tc/traffic_monitor.go b/lib/go-tc/traffic_monitor.go
index 2597ed5..f789214 100644
--- a/lib/go-tc/traffic_monitor.go
+++ b/lib/go-tc/traffic_monitor.go
@@ -97,6 +97,7 @@ type TMParameters struct {
HealthConnectionTimeout int `json:"health.connection.timeout"`
HealthPollingURL string `json:"health.polling.url"`
HealthPollingFormat string `json:"health.polling.format"`
+ HealthPollingType string `json:"health.polling.type"`
HistoryCount int `json:"history.count"`
MinFreeKbps int64
Thresholds map[string]HealthThreshold `json:"health_threshold"`
@@ -150,6 +151,22 @@ func (params *TMParameters) UnmarshalJSON(bytes []byte) (err error) {
}
}
+ if vi, ok := raw["health.polling.format"]; ok {
+ if v, ok := vi.(string); !ok {
+ return fmt.Errorf("Unmarshalling TMParameters health.polling.format expected string, got %v", vi)
+ } else {
+ params.HealthPollingFormat = v
+ }
+ }
+
+ if vi, ok := raw["health.polling.type"]; ok {
+ if v, ok := vi.(string); !ok {
+ return fmt.Errorf("Unmarshalling TMParameters health.polling.type expected string, got %v", vi)
+ } else {
+ params.HealthPollingType = v
+ }
+ }
+
if vi, ok := raw["history.count"]; ok {
if v, ok := vi.(float64); !ok {
return fmt.Errorf("Unmarshalling TMParameters history.count expected integer, got %v", vi)
diff --git a/traffic_monitor/fetcher/fetcher.go b/traffic_monitor/fetcher/fetcher.go
deleted file mode 100644
index e299a59..0000000
--- a/traffic_monitor/fetcher/fetcher.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package fetcher
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import (
- "fmt"
- "io/ioutil"
- "net/http"
- "time"
-
- "github.com/apache/trafficcontrol/lib/go-log"
- "github.com/apache/trafficcontrol/traffic_monitor/handler"
-)
-
-type Fetcher interface {
- Fetch(id string, url string, host string, format string, pollId uint64, pollFinishedChan chan<- uint64)
-}
-
-type HttpFetcher struct {
- Client *http.Client
- UserAgent string
- Headers map[string]string
- Handler handler.Handler
-}
-
-type Result struct {
- Source string
- Data []byte
- Error error
-}
-
-func (f HttpFetcher) Fetch(id string, url string, host string, format string, pollId uint64, pollFinishedChan chan<- uint64) {
- log.Debugf("poll %v %v fetch start\n", pollId, time.Now())
- req, err := http.NewRequest("GET", url, nil)
- // TODO: change this to use f.Headers. -jse
- req.Header.Set("User-Agent", f.UserAgent)
- req.Header.Set("Connection", "keep-alive")
- req.Host = host
- startReq := time.Now()
- response, err := f.Client.Do(req)
- reqEnd := time.Now()
- reqTime := reqEnd.Sub(startReq)
- defer func() {
- if response != nil && response.Body != nil {
- ioutil.ReadAll(response.Body) // TODO determine if necessary
- response.Body.Close()
- }
- }()
-
- if err == nil && response == nil {
- err = fmt.Errorf("err nil and response nil")
- }
- if err == nil && response != nil && (response.StatusCode < 200 || response.StatusCode > 299) {
- err = fmt.Errorf("bad status: %v", response.StatusCode)
- }
- if err != nil {
- err = fmt.Errorf("id %v url %v fetch error: %v", id, url, err)
- }
-
- if err == nil && response != nil {
- log.Debugf("poll %v %v fetch end\n", pollId, time.Now())
- f.Handler.Handle(id, response.Body, format, reqTime, reqEnd, err, pollId, pollFinishedChan)
- } else {
- f.Handler.Handle(id, nil, format, reqTime, reqEnd, err, pollId, pollFinishedChan)
- }
-}
diff --git a/traffic_monitor/manager/manager.go b/traffic_monitor/manager/manager.go
index d36b9e4..6ba9dc9 100644
--- a/traffic_monitor/manager/manager.go
+++ b/traffic_monitor/manager/manager.go
@@ -20,10 +20,8 @@ package manager
*/
import (
- "crypto/tls"
"fmt"
"io/ioutil"
- "net/http"
"os"
"os/signal"
@@ -44,12 +42,8 @@ import (
//
// Start starts the poller and handler goroutines
//
-func Start(opsConfigFile string, cfg config.Config, staticAppData config.StaticAppData, trafficMonitorConfigFileName string) error {
+func Start(opsConfigFile string, cfg config.Config, appData config.StaticAppData, trafficMonitorConfigFileName string) error {
toSession := towrap.ITrafficOpsSession(towrap.NewTrafficOpsSessionThreadsafe(nil, cfg.CRConfigHistoryCount))
- sharedClient := &http.Client{
- Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
- Timeout: cfg.HTTPTimeout,
- }
localStates := peer.NewCRStatesThreadsafe() // this is the local state as discoverer by this traffic_monitor
fetchCount := threadsafe.NewUint() // note this is the number of individual caches fetched from, not the number of times all the caches were polled.
@@ -59,12 +53,12 @@ func Start(opsConfigFile string, cfg config.Config, staticAppData config.StaticA
toData := todata.NewThreadsafe()
cacheHealthHandler := cache.NewHandler()
- cacheHealthPoller := poller.NewHTTP(cfg.CacheHealthPollingInterval, true, sharedClient, cacheHealthHandler, staticAppData.UserAgent)
+ cacheHealthPoller := poller.NewCache(cfg.CacheHealthPollingInterval, true, cacheHealthHandler, cfg, appData)
cacheStatHandler := cache.NewPrecomputeHandler(toData)
- cacheStatPoller := poller.NewHTTP(cfg.CacheStatPollingInterval, false, sharedClient, cacheStatHandler, staticAppData.UserAgent)
+ cacheStatPoller := poller.NewCache(cfg.CacheStatPollingInterval, false, cacheStatHandler, cfg, appData)
monitorConfigPoller := poller.NewMonitorConfig(cfg.MonitorConfigPollingInterval)
peerHandler := peer.NewHandler()
- peerPoller := poller.NewHTTP(cfg.PeerPollingInterval, false, sharedClient, peerHandler, staticAppData.UserAgent)
+ peerPoller := poller.NewCache(cfg.PeerPollingInterval, false, peerHandler, cfg, appData)
go monitorConfigPoller.Poll()
go cacheHealthPoller.Poll()
@@ -86,7 +80,7 @@ func Start(opsConfigFile string, cfg config.Config, staticAppData config.StaticA
monitorConfigPoller.IntervalChan,
cachesChanged,
cfg,
- staticAppData,
+ appData,
toSession,
toData,
)
@@ -142,7 +136,7 @@ func Start(opsConfigFile string, cfg config.Config, staticAppData config.StaticA
lastKbpsStats,
dsStats,
events,
- staticAppData,
+ appData,
cacheHealthPoller.Config.Interval,
lastHealthDurations,
fetchCount,
diff --git a/traffic_monitor/manager/monitorconfig.go b/traffic_monitor/manager/monitorconfig.go
index 1d2ab2f..d1f0af1 100644
--- a/traffic_monitor/manager/monitorconfig.go
+++ b/traffic_monitor/manager/monitorconfig.go
@@ -27,8 +27,8 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
- "github.com/apache/trafficcontrol/traffic_monitor/config"
"github.com/apache/trafficcontrol/traffic_monitor/cache"
+ "github.com/apache/trafficcontrol/traffic_monitor/config"
"github.com/apache/trafficcontrol/traffic_monitor/peer"
"github.com/apache/trafficcontrol/traffic_monitor/poller"
"github.com/apache/trafficcontrol/traffic_monitor/threadsafe"
@@ -120,9 +120,9 @@ func StartMonitorConfigManager(
monitorConfigPollChan <-chan poller.MonitorCfg,
localStates peer.CRStatesThreadsafe,
peerStates peer.CRStatesPeersThreadsafe,
- statURLSubscriber chan<- poller.HttpPollerConfig,
- healthURLSubscriber chan<- poller.HttpPollerConfig,
- peerURLSubscriber chan<- poller.HttpPollerConfig,
+ statURLSubscriber chan<- poller.CachePollerConfig,
+ healthURLSubscriber chan<- poller.CachePollerConfig,
+ peerURLSubscriber chan<- poller.CachePollerConfig,
toIntervalSubscriber chan<- time.Duration,
cachesChangeSubscriber chan<- struct{},
cfg config.Config,
@@ -190,9 +190,9 @@ func monitorConfigListen(
monitorConfigPollChan <-chan poller.MonitorCfg,
localStates peer.CRStatesThreadsafe,
peerStates peer.CRStatesPeersThreadsafe,
- statURLSubscriber chan<- poller.HttpPollerConfig,
- healthURLSubscriber chan<- poller.HttpPollerConfig,
- peerURLSubscriber chan<- poller.HttpPollerConfig,
+ statURLSubscriber chan<- poller.CachePollerConfig,
+ healthURLSubscriber chan<- poller.CachePollerConfig,
+ peerURLSubscriber chan<- poller.CachePollerConfig,
toIntervalSubscriber chan<- time.Duration,
cachesChangeSubscriber chan<- struct{},
cfg config.Config,
@@ -261,6 +261,12 @@ func monitorConfigListen(
log.Infof("health.polling.format for '%v' is empty, using default '%v'", srv.HostName, format)
}
+ pollType := monitorConfig.Profile[srv.Profile].Parameters.HealthPollingType
+ if pollType == "" {
+ pollType = poller.DefaultPollerType
+ log.Infof("health.polling.type for '%v' is empty, using default '%v'", srv.HostName, pollType)
+ }
+
r := strings.NewReplacer(
"${hostname}", srv.IP,
"${interface_name}", srv.InterfaceName,
@@ -275,10 +281,10 @@ func monitorConfigListen(
log.Warnln("profile " + srv.Profile + " health.connection.timeout Parameter is missing or zero, using default " + DefaultHealthConnectionTimeout.String())
}
- healthURLs[srv.HostName] = poller.PollConfig{URL: url, Host: srv.FQDN, Timeout: connTimeout, Format: format}
+ healthURLs[srv.HostName] = poller.PollConfig{URL: url, Host: srv.FQDN, Timeout: connTimeout, Format: format, PollType: pollType}
r = strings.NewReplacer("application=system", "application=")
statURL := r.Replace(url)
- statURLs[srv.HostName] = poller.PollConfig{URL: statURL, Host: srv.FQDN, Timeout: connTimeout, Format: format}
+ statURLs[srv.HostName] = poller.PollConfig{URL: statURL, Host: srv.FQDN, Timeout: connTimeout, Format: format, PollType: pollType}
}
peerSet := map[tc.TrafficMonitorName]struct{}{}
@@ -295,9 +301,9 @@ func monitorConfigListen(
peerSet[tc.TrafficMonitorName(srv.HostName)] = struct{}{}
}
- statURLSubscriber <- poller.HttpPollerConfig{Urls: statURLs, Interval: intervals.Stat, NoKeepAlive: intervals.StatNoKeepAlive}
- healthURLSubscriber <- poller.HttpPollerConfig{Urls: healthURLs, Interval: intervals.Health, NoKeepAlive: intervals.HealthNoKeepAlive}
- peerURLSubscriber <- poller.HttpPollerConfig{Urls: peerURLs, Interval: intervals.Peer, NoKeepAlive: intervals.PeerNoKeepAlive}
+ statURLSubscriber <- poller.CachePollerConfig{Urls: statURLs, Interval: intervals.Stat, NoKeepAlive: intervals.StatNoKeepAlive}
+ healthURLSubscriber <- poller.CachePollerConfig{Urls: healthURLs, Interval: intervals.Health, NoKeepAlive: intervals.HealthNoKeepAlive}
+ peerURLSubscriber <- poller.CachePollerConfig{Urls: peerURLs, Interval: intervals.Peer, NoKeepAlive: intervals.PeerNoKeepAlive}
toIntervalSubscriber <- intervals.TO
peerStates.SetTimeout((intervals.Peer + cfg.HTTPTimeout) * 2)
peerStates.SetPeers(peerSet)
diff --git a/traffic_monitor/poller/cache.go b/traffic_monitor/poller/cache.go
new file mode 100644
index 0000000..930e693
--- /dev/null
+++ b/traffic_monitor/poller/cache.go
@@ -0,0 +1,241 @@
+package poller
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "bytes"
+ "io"
+ "math/rand"
+ "runtime"
+ "sync/atomic"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-log"
+ "github.com/apache/trafficcontrol/traffic_monitor/config"
+ "github.com/apache/trafficcontrol/traffic_monitor/handler"
+)
+
+type CachePoller struct {
+ Config CachePollerConfig
+ ConfigChannel chan CachePollerConfig
+ TickChan chan uint64
+ GlobalContexts map[string]interface{}
+ Handler handler.Handler
+}
+
+type PollConfig struct {
+ URL string
+ Host string
+ Timeout time.Duration
+ Format string
+ PollType string
+}
+
+type CachePollerConfig struct {
+ Urls map[string]PollConfig
+ Interval time.Duration
+ NoKeepAlive bool
+}
+
+// NewHTTP creates and returns a new CachePoller.
+// If tick is false, CachePoller.TickChan() will return nil.
+// TODO: rename to not "http", since it's now pluggable
+func NewCache(
+ interval time.Duration,
+ tick bool,
+ handler handler.Handler,
+ cfg config.Config,
+ appData config.StaticAppData,
+) CachePoller {
+ var tickChan chan uint64
+ if tick {
+ tickChan = make(chan uint64)
+ }
+ return CachePoller{
+ TickChan: tickChan,
+ ConfigChannel: make(chan CachePollerConfig),
+ Config: CachePollerConfig{
+ Interval: interval,
+ },
+ GlobalContexts: GetGlobalContexts(cfg, appData),
+ Handler: handler,
+ }
+}
+
+var pollNum uint64
+
+type CachePollInfo struct {
+ NoKeepAlive bool
+ Interval time.Duration
+ ID string
+ PollConfig
+}
+
+func (p CachePoller) Poll() {
+ killChans := map[string]chan<- struct{}{}
+ for newConfig := range p.ConfigChannel {
+ deletions, additions := diffConfigs(p.Config, newConfig)
+ for _, id := range deletions {
+ killChan := killChans[id]
+ go func() { killChan <- struct{}{} }() // go - we don't want to wait for old polls to die.
+ delete(killChans, id)
+ }
+ for _, info := range additions {
+ kill := make(chan struct{})
+ killChans[info.ID] = kill
+
+ if _, ok := pollers[info.PollType]; !ok {
+ if info.PollType != "" { // don't warn for missing parameters
+ log.Warnln("CachePoller.Poll: poll type '" + info.PollType + "' not found, using default poll type '" + DefaultPollerType + "'")
+ }
+ info.PollType = DefaultPollerType
+ }
+ pollerObj := pollers[info.PollType]
+ pollerCfg := PollerConfig{
+ URL: info.URL,
+ Host: info.Host,
+ Timeout: info.Timeout,
+ NoKeepAlive: info.NoKeepAlive,
+ PollerID: info.ID,
+ }
+ pollerCtx := interface{}(nil)
+ if pollerObj.Init != nil {
+ pollerCtx = pollerObj.Init(pollerCfg, p.GlobalContexts[info.PollType])
+ }
+ go poller(info.Interval, info.ID, info.URL, info.Host, info.Format, p.Handler, pollerObj.Poll, pollerCtx, kill)
+ }
+ p.Config = newConfig
+ }
+}
+
+func mustDie(die <-chan struct{}) bool {
+ select {
+ case <-die:
+ return true
+ default:
+ }
+ return false
+}
+
+// TODO iterationCount and/or p.TickChan?
+func poller(
+ interval time.Duration,
+ id string,
+ url string,
+ host string,
+ format string,
+ handler handler.Handler,
+ pollFunc PollerFunc,
+ pollCtx interface{},
+ die <-chan struct{},
+) {
+ pollSpread := time.Duration(rand.Float64()*float64(interval/time.Nanosecond)) * time.Nanosecond
+ time.Sleep(pollSpread)
+ tick := time.NewTicker(interval)
+ lastTime := time.Now()
+ for {
+ select {
+ case <-tick.C:
+ realInterval := time.Now().Sub(lastTime)
+ if realInterval > interval+(time.Millisecond*100) {
+ log.Debugf("Intended Duration: %v Actual Duration: %v\n", interval, realInterval)
+ }
+ lastTime = time.Now()
+
+ pollID := atomic.AddUint64(&pollNum, 1)
+ pollFinishedChan := make(chan uint64)
+ log.Debugf("poll %v %v start\n", pollID, time.Now())
+ bts, reqEnd, reqTime, err := pollFunc(pollCtx, url, host, pollID)
+ rdr := io.Reader(nil)
+ if bts != nil {
+ rdr = bytes.NewReader(bts) // TODO change handler to take bytes? Benchmark?
+ } else {
+ }
+ log.Debugf("poll %v %v poller end\n", pollID, time.Now())
+ go handler.Handle(id, rdr, format, reqTime, reqEnd, err, pollID, pollFinishedChan)
+ <-pollFinishedChan
+ case <-die:
+ tick.Stop()
+ return
+ }
+ }
+}
+
+// diffConfigs takes the old and new configs, and returns a list of deleted IDs, and a list of new polls to do
+func diffConfigs(old CachePollerConfig, new CachePollerConfig) ([]string, []CachePollInfo) {
+ deletions := []string{}
+ additions := []CachePollInfo{}
+
+ if old.Interval != new.Interval || old.NoKeepAlive != new.NoKeepAlive {
+ for id, _ := range old.Urls {
+ deletions = append(deletions, id)
+ }
+ for id, pollCfg := range new.Urls {
+ additions = append(additions, CachePollInfo{
+ Interval: new.Interval,
+ NoKeepAlive: new.NoKeepAlive,
+ ID: id,
+ PollConfig: pollCfg,
+ })
+ }
+ return deletions, additions
+ }
+
+ for id, oldPollCfg := range old.Urls {
+ newPollCfg, newIdExists := new.Urls[id]
+ if !newIdExists {
+ deletions = append(deletions, id)
+ } else if newPollCfg != oldPollCfg {
+ deletions = append(deletions, id)
+ additions = append(additions, CachePollInfo{
+ Interval: new.Interval,
+ NoKeepAlive: new.NoKeepAlive,
+ ID: id,
+ PollConfig: newPollCfg,
+ })
+ }
+ }
+
+ for id, newPollCfg := range new.Urls {
+ _, oldIdExists := old.Urls[id]
+ if !oldIdExists {
+ additions = append(additions, CachePollInfo{
+ Interval: new.Interval,
+ NoKeepAlive: new.NoKeepAlive,
+ ID: id,
+ PollConfig: newPollCfg,
+ })
+ }
+ }
+
+ return deletions, additions
+}
+
+func stacktrace() []byte {
+ initialBufSize := 1024
+ buf := make([]byte, initialBufSize)
+ for {
+ n := runtime.Stack(buf, true)
+ if n < len(buf) {
+ return buf[:n]
+ }
+ buf = make([]byte, len(buf)*2)
+ }
+}
diff --git a/traffic_monitor/poller/monitorconfig.go b/traffic_monitor/poller/monitorconfig.go
new file mode 100644
index 0000000..40e2108
--- /dev/null
+++ b/traffic_monitor/poller/monitorconfig.go
@@ -0,0 +1,119 @@
+package poller
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "os"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-log"
+ "github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/traffic_monitor/handler"
+ "github.com/apache/trafficcontrol/traffic_monitor/towrap" // TODO move to common
+)
+
+type MonitorCfg struct {
+ CDN string
+ Cfg tc.TrafficMonitorConfigMap
+}
+
+type MonitorConfigPoller struct {
+ Session towrap.ITrafficOpsSession
+ SessionChannel chan towrap.ITrafficOpsSession
+ ConfigChannel chan MonitorCfg
+ OpsConfigChannel chan handler.OpsConfig
+ Interval time.Duration
+ IntervalChan chan time.Duration
+ OpsConfig handler.OpsConfig
+}
+
+// Creates and returns a new CachePoller.
+// If tick is false, CachePoller.TickChan() will return nil
+func NewMonitorConfig(interval time.Duration) MonitorConfigPoller {
+ return MonitorConfigPoller{
+ Interval: interval,
+ SessionChannel: make(chan towrap.ITrafficOpsSession),
+ // ConfigChannel MUST have a buffer size 1, to make the nonblocking writeConfig work
+ ConfigChannel: make(chan MonitorCfg, 1),
+ OpsConfigChannel: make(chan handler.OpsConfig),
+ IntervalChan: make(chan time.Duration),
+ }
+}
+
+// writeConfig writes the given config to the Config chan. This is nonblocking, and immediately returns.
+// Because readers only ever want the latest config, if nobody has read the previous write, we remove it. Since the config chan is buffered size 1, this function is therefore asynchronous.
+func (p MonitorConfigPoller) writeConfig(cfg MonitorCfg) {
+ for {
+ select {
+ case p.ConfigChannel <- cfg:
+ return // return after successfully writing.
+ case <-p.ConfigChannel:
+ // if the channel buffer was full, read, then loop and try to write again
+ }
+ }
+}
+
+func (p MonitorConfigPoller) Poll() {
+ tick := time.NewTicker(p.Interval)
+ defer tick.Stop()
+ defer func() {
+ if err := recover(); err != nil {
+ log.Errorf("MonitorConfigPoller panic: %v\n", err)
+ } else {
+ log.Errorf("MonitorConfigPoller failed without panic\n")
+ }
+ log.Errorf("%s\n", stacktrace())
+ os.Exit(1) // The Monitor can't run without a MonitorConfigPoller
+ }()
+ for {
+ // Every case MUST be asynchronous and non-blocking, to prevent livelocks. If a chan must be written to, it must either be buffered AND remove existing values, or be written to in a goroutine.
+ select {
+ case opsConfig := <-p.OpsConfigChannel:
+ log.Infof("MonitorConfigPoller: received new opsConfig: %v\n", opsConfig)
+ p.OpsConfig = opsConfig
+ case session := <-p.SessionChannel:
+ log.Infof("MonitorConfigPoller: received new session: %v\n", session)
+ p.Session = session
+ case i := <-p.IntervalChan:
+ if i == p.Interval {
+ continue
+ }
+ log.Infof("MonitorConfigPoller: received new interval: %v\n", i)
+ if i < 0 {
+ log.Errorf("MonitorConfigPoller: received negative interval: %v; ignoring\n", i)
+ continue
+ }
+ p.Interval = i
+ tick.Stop()
+ tick = time.NewTicker(p.Interval)
+ case <-tick.C:
+ if p.Session == nil || p.OpsConfig.CdnName == "" {
+ log.Warnln("MonitorConfigPoller: skipping this iteration, Session is nil")
+ continue
+ }
+ monitorConfig, err := p.Session.TrafficMonitorConfigMap(p.OpsConfig.CdnName)
+ if err != nil {
+ log.Errorf("MonitorConfigPoller: %s\n %v\n", err, monitorConfig)
+ continue
+ }
+ p.writeConfig(MonitorCfg{CDN: p.OpsConfig.CdnName, Cfg: *monitorConfig})
+ }
+ }
+}
diff --git a/traffic_monitor/poller/poller.go b/traffic_monitor/poller/poller.go
deleted file mode 100644
index ed12613..0000000
--- a/traffic_monitor/poller/poller.go
+++ /dev/null
@@ -1,324 +0,0 @@
-package poller
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import (
- "math/rand"
- "net/http"
- "os"
- "runtime"
- "sync/atomic"
- "time"
-
- "github.com/apache/trafficcontrol/lib/go-log"
- tc "github.com/apache/trafficcontrol/lib/go-tc"
- "github.com/apache/trafficcontrol/traffic_monitor/fetcher"
- "github.com/apache/trafficcontrol/traffic_monitor/handler"
- "github.com/apache/trafficcontrol/traffic_monitor/towrap" // TODO move to common
-)
-
-type Poller interface {
- Poll()
-}
-
-type HttpPoller struct {
- Config HttpPollerConfig
- ConfigChannel chan HttpPollerConfig
- FetcherTemplate fetcher.HttpFetcher // FetcherTemplate has all the constant settings, and is copied to create fetchers with custom HTTP client timeouts.
- TickChan chan uint64
-}
-
-type PollConfig struct {
- URL string
- Host string
- Timeout time.Duration
- Handler handler.Handler
- Format string
-}
-
-type HttpPollerConfig struct {
- Urls map[string]PollConfig
- Interval time.Duration
- NoKeepAlive bool
-}
-
-// NewHTTP creates and returns a new HttpPoller.
-// If tick is false, HttpPoller.TickChan() will return nil.
-func NewHTTP(
- interval time.Duration,
- tick bool,
- httpClient *http.Client,
- fetchHandler handler.Handler,
- userAgent string,
-) HttpPoller {
- var tickChan chan uint64
- if tick {
- tickChan = make(chan uint64)
- }
- return HttpPoller{
- TickChan: tickChan,
- ConfigChannel: make(chan HttpPollerConfig),
- Config: HttpPollerConfig{
- Interval: interval,
- },
- FetcherTemplate: fetcher.HttpFetcher{
- Handler: fetchHandler,
- Client: httpClient,
- UserAgent: userAgent,
- },
- }
-}
-
-type MonitorCfg struct {
- CDN string
- Cfg tc.TrafficMonitorConfigMap
-}
-
-type MonitorConfigPoller struct {
- Session towrap.ITrafficOpsSession
- SessionChannel chan towrap.ITrafficOpsSession
- ConfigChannel chan MonitorCfg
- OpsConfigChannel chan handler.OpsConfig
- Interval time.Duration
- IntervalChan chan time.Duration
- OpsConfig handler.OpsConfig
-}
-
-// Creates and returns a new HttpPoller.
-// If tick is false, HttpPoller.TickChan() will return nil
-func NewMonitorConfig(interval time.Duration) MonitorConfigPoller {
- return MonitorConfigPoller{
- Interval: interval,
- SessionChannel: make(chan towrap.ITrafficOpsSession),
- // ConfigChannel MUST have a buffer size 1, to make the nonblocking writeConfig work
- ConfigChannel: make(chan MonitorCfg, 1),
- OpsConfigChannel: make(chan handler.OpsConfig),
- IntervalChan: make(chan time.Duration),
- }
-}
-
-// writeConfig writes the given config to the Config chan. This is nonblocking, and immediately returns.
-// Because readers only ever want the latest config, if nobody has read the previous write, we remove it. Since the config chan is buffered size 1, this function is therefore asynchronous.
-func (p MonitorConfigPoller) writeConfig(cfg MonitorCfg) {
- for {
- select {
- case p.ConfigChannel <- cfg:
- return // return after successfully writing.
- case <-p.ConfigChannel:
- // if the channel buffer was full, read, then loop and try to write again
- }
- }
-}
-
-func (p MonitorConfigPoller) Poll() {
- tick := time.NewTicker(p.Interval)
- defer tick.Stop()
- defer func() {
- if err := recover(); err != nil {
- log.Errorf("MonitorConfigPoller panic: %v\n", err)
- } else {
- log.Errorf("MonitorConfigPoller failed without panic\n")
- }
- log.Errorf("%s\n", stacktrace())
- os.Exit(1) // The Monitor can't run without a MonitorConfigPoller
- }()
- for {
- // Every case MUST be asynchronous and non-blocking, to prevent livelocks. If a chan must be written to, it must either be buffered AND remove existing values, or be written to in a goroutine.
- select {
- case opsConfig := <-p.OpsConfigChannel:
- log.Infof("MonitorConfigPoller: received new opsConfig: %v\n", opsConfig)
- p.OpsConfig = opsConfig
- case session := <-p.SessionChannel:
- log.Infof("MonitorConfigPoller: received new session: %v\n", session)
- p.Session = session
- case i := <-p.IntervalChan:
- if i == p.Interval {
- continue
- }
- log.Infof("MonitorConfigPoller: received new interval: %v\n", i)
- if i < 0 {
- log.Errorf("MonitorConfigPoller: received negative interval: %v; ignoring\n", i)
- continue
- }
- p.Interval = i
- tick.Stop()
- tick = time.NewTicker(p.Interval)
- case <-tick.C:
- if p.Session == nil || p.OpsConfig.CdnName == "" {
- log.Warnln("MonitorConfigPoller: skipping this iteration, Session is nil")
- continue
- }
- monitorConfig, err := p.Session.TrafficMonitorConfigMap(p.OpsConfig.CdnName)
- if err != nil {
- log.Errorf("MonitorConfigPoller: %s\n %v\n", err, monitorConfig)
- continue
- }
- p.writeConfig(MonitorCfg{CDN: p.OpsConfig.CdnName, Cfg: *monitorConfig})
- }
- }
-}
-
-var debugPollNum uint64
-
-type HTTPPollInfo struct {
- NoKeepAlive bool
- Interval time.Duration
- ID string
- PollConfig
-}
-
-func (p HttpPoller) Poll() {
- // iterationCount := uint64(0)
- // iterationCount++ // on tick<:
- // case p.TickChan <- iterationCount:
- killChans := map[string]chan<- struct{}{}
- for newConfig := range p.ConfigChannel {
- deletions, additions := diffConfigs(p.Config, newConfig)
- for _, id := range deletions {
- killChan := killChans[id]
- go func() { killChan <- struct{}{} }() // go - we don't want to wait for old polls to die.
- delete(killChans, id)
- }
- for _, info := range additions {
- kill := make(chan struct{})
- killChans[info.ID] = kill
-
- fetcher := p.FetcherTemplate
- if info.Timeout != 0 || info.NoKeepAlive { // if the timeout isn't explicitly set, use the template value.
- c := *fetcher.Client
- fetcher.Client = &c // copy the client, so we don't change other fetchers.
- if info.Timeout != 0 {
- fetcher.Client.Timeout = info.Timeout
- }
- if info.NoKeepAlive {
- transportI := http.DefaultTransport
- transport, ok := transportI.(*http.Transport)
- if !ok {
- log.Errorf("failed to set NoKeepAlive for '%v': http.DefaultTransport expected type *http.Transport actual %T\n", info.URL, transportI)
- } else {
- transport.DisableKeepAlives = info.NoKeepAlive
- fetcher.Client.Transport = transport
- log.Infof("Setting transport.DisableKeepAlives %v for %v\n", transport.DisableKeepAlives, info.URL)
- }
- }
- }
- go poller(info.Interval, info.ID, info.URL, info.Host, info.Format, fetcher, kill)
- }
- p.Config = newConfig
- }
-}
-
-func mustDie(die <-chan struct{}) bool {
- select {
- case <-die:
- return true
- default:
- }
- return false
-}
-
-// TODO iterationCount and/or p.TickChan?
-func poller(interval time.Duration, id string, url string, host string, format string, fetcher fetcher.Fetcher, die <-chan struct{}) {
- pollSpread := time.Duration(rand.Float64()*float64(interval/time.Nanosecond)) * time.Nanosecond
- time.Sleep(pollSpread)
- tick := time.NewTicker(interval)
- lastTime := time.Now()
- for {
- select {
- case <-tick.C:
- realInterval := time.Now().Sub(lastTime)
- if realInterval > interval+(time.Millisecond*100) {
- log.Debugf("Intended Duration: %v Actual Duration: %v\n", interval, realInterval)
- }
- lastTime = time.Now()
-
- pollId := atomic.AddUint64(&debugPollNum, 1)
- pollFinishedChan := make(chan uint64)
- log.Debugf("poll %v %v start\n", pollId, time.Now())
- go fetcher.Fetch(id, url, host, format, pollId, pollFinishedChan) // TODO persist fetcher, with its own die chan?
- <-pollFinishedChan
- case <-die:
- tick.Stop()
- return
- }
- }
-}
-
-// diffConfigs takes the old and new configs, and returns a list of deleted IDs, and a list of new polls to do
-func diffConfigs(old HttpPollerConfig, new HttpPollerConfig) ([]string, []HTTPPollInfo) {
- deletions := []string{}
- additions := []HTTPPollInfo{}
-
- if old.Interval != new.Interval || old.NoKeepAlive != new.NoKeepAlive {
- for id, _ := range old.Urls {
- deletions = append(deletions, id)
- }
- for id, pollCfg := range new.Urls {
- additions = append(additions, HTTPPollInfo{
- Interval: new.Interval,
- NoKeepAlive: new.NoKeepAlive,
- ID: id,
- PollConfig: pollCfg,
- })
- }
- return deletions, additions
- }
-
- for id, oldPollCfg := range old.Urls {
- newPollCfg, newIdExists := new.Urls[id]
- if !newIdExists {
- deletions = append(deletions, id)
- } else if newPollCfg != oldPollCfg {
- deletions = append(deletions, id)
- additions = append(additions, HTTPPollInfo{
- Interval: new.Interval,
- NoKeepAlive: new.NoKeepAlive,
- ID: id,
- PollConfig: newPollCfg,
- })
- }
- }
-
- for id, newPollCfg := range new.Urls {
- _, oldIdExists := old.Urls[id]
- if !oldIdExists {
- additions = append(additions, HTTPPollInfo{
- Interval: new.Interval,
- NoKeepAlive: new.NoKeepAlive,
- ID: id,
- PollConfig: newPollCfg,
- })
- }
- }
-
- return deletions, additions
-}
-
-func stacktrace() []byte {
- initialBufSize := 1024
- buf := make([]byte, initialBufSize)
- for {
- n := runtime.Stack(buf, true)
- if n < len(buf) {
- return buf[:n]
- }
- buf = make([]byte, len(buf)*2)
- }
-}
diff --git a/traffic_monitor/poller/poller_type_http.go b/traffic_monitor/poller/poller_type_http.go
new file mode 100644
index 0000000..cc45818
--- /dev/null
+++ b/traffic_monitor/poller/poller_type_http.go
@@ -0,0 +1,127 @@
+package poller
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-log"
+ "github.com/apache/trafficcontrol/traffic_monitor/config"
+)
+
+const PollerTypeHTTP = "http"
+
+func init() {
+ AddPollerType(PollerTypeHTTP, httpGlobalInit, httpInit, httpPoll)
+}
+
+func httpGlobalInit(cfg config.Config, appData config.StaticAppData) interface{} {
+ sharedClient := &http.Client{
+ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
+ Timeout: cfg.HTTPTimeout,
+ }
+ return &HTTPPollGlobalCtx{
+ UserAgent: appData.UserAgent,
+ Client: sharedClient,
+ }
+}
+
+func httpInit(cfg PollerConfig, globalCtxI interface{}) interface{} {
+ gctx := (globalCtxI).(*HTTPPollGlobalCtx)
+
+ if cfg.Timeout != 0 || cfg.NoKeepAlive { // if the timeout isn't explicitly set, use the template value.
+ clientCopy := *gctx.Client
+ gctx.Client = &clientCopy // copy the client, so it's reused by pollers who DO use the default timeout/keepalive
+ if cfg.Timeout != 0 {
+ gctx.Client.Timeout = cfg.Timeout
+ }
+ if cfg.NoKeepAlive {
+ transportI := http.DefaultTransport
+ transport, ok := transportI.(*http.Transport)
+ if !ok {
+ log.Errorf("failed to set NoKeepAlive for '%v': http.DefaultTransport expected type *http.Transport actual %T\n", cfg.URL, transportI)
+ } else {
+ transport.DisableKeepAlives = cfg.NoKeepAlive
+ gctx.Client.Transport = transport
+ log.Infof("Setting transport.DisableKeepAlives %v for %v\n", transport.DisableKeepAlives, cfg.URL)
+ }
+ }
+ }
+
+ return &HTTPPollCtx{
+ Client: gctx.Client,
+ UserAgent: gctx.UserAgent,
+ NoKeepAlive: cfg.NoKeepAlive,
+ URL: cfg.URL,
+ Host: cfg.Host,
+ PollerID: cfg.PollerID,
+ }
+}
+
+type HTTPPollGlobalCtx struct {
+ Client *http.Client
+ UserAgent string
+}
+
+type HTTPPollCtx struct {
+ Client *http.Client
+ UserAgent string
+ NoKeepAlive bool
+ URL string
+ Host string
+ PollerID string
+}
+
+func httpPoll(ctxI interface{}, url string, host string, pollID uint64) ([]byte, time.Time, time.Duration, error) {
+ ctx := (ctxI).(*HTTPPollCtx)
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, time.Now(), 0, errors.New("creating HTTP request: " + err.Error())
+ }
+ req.Header.Set("User-Agent", ctx.UserAgent)
+ if !ctx.NoKeepAlive {
+ req.Header.Set("Connection", "keep-alive")
+ }
+ req.Host = host
+ startReq := time.Now()
+ resp, err := ctx.Client.Do(req)
+ reqEnd := time.Now()
+ reqTime := reqEnd.Sub(startReq) // note this is essentially the roundtrip, not the body transfer time.
+ if err != nil {
+ return nil, reqEnd, reqTime, fmt.Errorf("id %v url %v fetch error: %v", ctx.PollerID, url, err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode < 200 || resp.StatusCode > 299 {
+ return nil, reqEnd, reqTime, fmt.Errorf("id %v url %v fetch error: bad HTTP status: %v", ctx.PollerID, url, resp.StatusCode)
+ }
+
+ bts, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, reqEnd, reqTime, fmt.Errorf("id %v url %v fetch error: reading body: %v", ctx.PollerID, url, err)
+ }
+
+ return bts, reqEnd, reqTime, nil
+}
diff --git a/traffic_monitor/poller/poller_type_noop.go b/traffic_monitor/poller/poller_type_noop.go
new file mode 100644
index 0000000..d63ee1e
--- /dev/null
+++ b/traffic_monitor/poller/poller_type_noop.go
@@ -0,0 +1,34 @@
+package poller
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "time"
+)
+
+const PollerTypeNOOP = "noop"
+
+func init() {
+ AddPollerType(PollerTypeNOOP, nil, nil, noopPoll)
+}
+
+func noopPoll(ctx interface{}, url string, host string, pollID uint64) ([]byte, time.Time, time.Duration, error) {
+ return nil, time.Now(), 0, nil
+}
diff --git a/traffic_monitor/poller/poller_types.go b/traffic_monitor/poller/poller_types.go
new file mode 100644
index 0000000..937de13
--- /dev/null
+++ b/traffic_monitor/poller/poller_types.go
@@ -0,0 +1,75 @@
+package poller
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "time"
+
+ "github.com/apache/trafficcontrol/traffic_monitor/config"
+)
+
+const DefaultPollerType = PollerTypeHTTP
+
+// Poller is a particular type of cache stat poller. Examples are HTTP, TCP, or NFS files. It only polls and returns bytes received, it does not do any parsing. For stat parsing, see traffic_monitor/cache/stats_types.go.
+type PollerType struct {
+ GlobalInit PollerGlobalInitFunc
+ Init PollerInitFunc
+ Poll PollerFunc
+}
+
+// PollerConfig is the data given to cache pollers when they're initialized.
+type PollerConfig struct {
+ URL string
+ Host string
+ Timeout time.Duration
+ NoKeepAlive bool
+ PollerID string
+}
+
+// PollerGlobalInit performs global initialization, and returns a global context object.
+// For example, a poller might create a threadsafe client object, which can be used concurrently by all pollers.
+type PollerGlobalInitFunc func(cfg config.Config, appData config.StaticAppData) interface{}
+
+// PollerInit performs initialization for a specific poller. It takes the global context created by the poller's GlobalInit.
+// For example, a poller might create a template client in its GlobalInit, and then use that to create a non-threadsafe client for each concurrent poller.
+type PollerInitFunc func(cfg PollerConfig, globalCtx interface{}) interface{}
+
+// pollers holds the functions for polling caches. This is not const, because Go doesn't allow constant maps. This is populated on startup, and MUST NOT be modified after startup.
+var pollers = map[string]PollerType{}
+
+// PollerFunc polls a cache. It takes the global context created by this Poller's GlobalInit, and the poller-specific context created by this poller's Init. It returns the response bytes, the time the request finished, the length of time the request took, and any error.
+// If the PollerFunc needs the global context object, the Init func should embed it in the context object it returns. If Init is nil, the global context will be given to the poller.
+type PollerFunc func(ctx interface{}, url string, host string, pollID uint64) ([]byte, time.Time, time.Duration, error)
+
+// AddPollerType adds a poller with the given name, and the given init and poll funcs. The globalInit and init funcs may be nil; poller MUST NOT be nil.
+func AddPollerType(name string, globalInit PollerGlobalInitFunc, init PollerInitFunc, poller PollerFunc) {
+ pollers[name] = PollerType{GlobalInit: globalInit, Init: init, Poll: poller}
+}
+
+// GetGlobalContexts returns the global contexts corresponding to the registered pollers
+func GetGlobalContexts(cfg config.Config, appData config.StaticAppData) map[string]interface{} {
+ ctxs := map[string]interface{}{}
+ for pollerName, poller := range pollers {
+ if poller.GlobalInit != nil {
+ ctxs[pollerName] = poller.GlobalInit(cfg, appData)
+ }
+ }
+ return ctxs
+}
[trafficcontrol] 02/02: Add TM noop stat parser type
Posted by dg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dgelinas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
commit 01fab085b9d6844c4781bf722544c1c9c8f4cc66
Author: Robert Butts <ro...@apache.org>
AuthorDate: Sun Aug 12 16:56:53 2018 -0600
Add TM noop stat parser type
---
traffic_monitor/cache/cache.go | 11 ++-----
traffic_monitor/cache/stats_type_astats.go | 9 ++++--
traffic_monitor/cache/stats_type_noop.go | 50 ++++++++++++++++++++++++++++++
3 files changed, 59 insertions(+), 11 deletions(-)
diff --git a/traffic_monitor/cache/cache.go b/traffic_monitor/cache/cache.go
index d40a340..5e01015 100644
--- a/traffic_monitor/cache/cache.go
+++ b/traffic_monitor/cache/cache.go
@@ -272,7 +272,7 @@ func StatsMarshall(statResultHistory ResultStatHistory, statInfo ResultInfoHisto
}
// Handle handles results fetched from a cache, parsing the raw Reader data and passing it along to a chan for further processing.
-func (handler Handler) Handle(id string, r io.Reader, format string, reqTime time.Duration, reqEnd time.Time, reqErr error, pollID uint64, pollFinished chan<- uint64) {
+func (handler Handler) Handle(id string, rdr io.Reader, format string, reqTime time.Duration, reqEnd time.Time, reqErr error, pollID uint64, pollFinished chan<- uint64) {
log.Debugf("poll %v %v (format '%v') handle start\n", pollID, time.Now(), format)
result := Result{
ID: tc.CacheName(id),
@@ -289,13 +289,6 @@ func (handler Handler) Handle(id string, r io.Reader, format string, reqTime tim
return
}
- if r == nil {
- log.Warnf("%v handle reader nil\n", id)
- result.Error = fmt.Errorf("handler got nil reader")
- handler.resultChan <- result
- return
- }
-
statDecoder, ok := StatsTypeDecoders[format]
if !ok {
log.Errorf("Handler cache '%s' stat type '%s' not found! Returning handle error for this cache poll.\n", id, format)
@@ -305,7 +298,7 @@ func (handler Handler) Handle(id string, r io.Reader, format string, reqTime tim
}
decodeErr := error(nil)
- if decodeErr, result.Astats.Ats, result.Astats.System = statDecoder.Parse(result.ID, r); decodeErr != nil {
+ if decodeErr, result.Astats.Ats, result.Astats.System = statDecoder.Parse(result.ID, rdr); decodeErr != nil {
log.Warnf("%s decode error '%v'\n", id, decodeErr)
result.Error = decodeErr
handler.resultChan <- result
diff --git a/traffic_monitor/cache/stats_type_astats.go b/traffic_monitor/cache/stats_type_astats.go
index 81588f5..11a1917 100644
--- a/traffic_monitor/cache/stats_type_astats.go
+++ b/traffic_monitor/cache/stats_type_astats.go
@@ -30,6 +30,7 @@ package cache
import (
"encoding/json"
+ "errors"
"fmt"
"io"
"strconv"
@@ -45,9 +46,13 @@ func init() {
AddStatsType("astats", astatsParse, astatsPrecompute)
}
-func astatsParse(cache tc.CacheName, r io.Reader) (error, map[string]interface{}, AstatsSystem) {
+func astatsParse(cache tc.CacheName, rdr io.Reader) (error, map[string]interface{}, AstatsSystem) {
+ if rdr == nil {
+ log.Warnln(string(cache) + " handle reader nil")
+ return errors.New("handler got nil reader"), nil, AstatsSystem{}
+ }
astats := Astats{}
- err := json.NewDecoder(r).Decode(&astats)
+ err := json.NewDecoder(rdr).Decode(&astats)
return err, astats.Ats, astats.System
}
diff --git a/traffic_monitor/cache/stats_type_noop.go b/traffic_monitor/cache/stats_type_noop.go
new file mode 100644
index 0000000..9610ed7
--- /dev/null
+++ b/traffic_monitor/cache/stats_type_noop.go
@@ -0,0 +1,50 @@
+package cache
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// stats_type_noop is a no-op parser designed to work with the the noop poller, to report caches as healthy without actually polling them.
+
+import (
+ "io"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/traffic_monitor/dsdata"
+ "github.com/apache/trafficcontrol/traffic_monitor/todata"
+)
+
+const StatsTypeNOOP = "noop"
+
+func init() {
+ AddStatsType(StatsTypeNOOP, noopParse, noopPrecompute)
+}
+
+func noopParse(cache tc.CacheName, r io.Reader) (error, map[string]interface{}, AstatsSystem) {
+ // we need to make a fake system, so the health parse succeeds
+ return nil, map[string]interface{}{}, AstatsSystem{
+ ProcLoadavg: "0.10 0.05 0.05 1/1000 30000",
+ ProcNetDev: "bond0:10000 10000 0 0 0 0 0 1000 100000 1000000 0 0 0 0 0 0",
+ InfSpeed: 20000,
+ InfName: "bond0",
+ }
+}
+
+func noopPrecompute(cache tc.CacheName, toData todata.TOData, rawStats map[string]interface{}, system AstatsSystem) PrecomputedData {
+ return PrecomputedData{DeliveryServiceStats: map[tc.DeliveryServiceName]dsdata.Stat{}}
+}