You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@trafficcontrol.apache.org by GitBox <gi...@apache.org> on 2018/12/14 04:03:31 UTC
[GitHub] rob05c closed pull request #3053: Traffic Monitor Disk Support
rob05c closed pull request #3053: Traffic Monitor Disk Support
URL: https://github.com/apache/trafficcontrol/pull/3053
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/docs/source/development/traffic_monitor.rst b/docs/source/development/traffic_monitor.rst
index 4be0fdb59..a835d44d7 100644
--- a/docs/source/development/traffic_monitor.rst
+++ b/docs/source/development/traffic_monitor.rst
@@ -192,6 +192,14 @@ Shared Data
-----------
Processed and aggregated data must be shared between the end of the stat and health processing pipelines, and HTTP requests. The CSP paradigm of idiomatic Go does not work efficiently with storing and sharing state. While not idiomatic Go, shared mutexed data structures are faster and simpler than CSP manager microthreads for each data object. Traffic Monitor has many thread-safe shared data types and objects. All shared data objects can be seen in ``manager/manager.go:Start()``, where they are created and passed to the various pipeline stage microthreads that need them. Their respective types all include the word ``Threadsafe``, and can be found in ``traffic_monitor/threadsafe/`` as well as, for dependency reasons, various appropriate directories. Currently, all thread-safe shared data types use mutexes. In the future, these could be changed to lock-free or wait-free structures, if the performance needs outweighed the readability and correctness costs. They could also easily be changed to internally be manager microthreads and channels, if being idiomatic were deemed more important than readability or performance.
+Disk Backup
+------------
+The traffic monitor config and CR config are both stored as backup files (tmconfig.backup and crconfig.backup or what ever you set the values to in the config file). This allows the monitor to come up and continue serving even if traffic ops
+is down. These files are updated any time a valid config is received from traffic ops, so if traffic ops goes down and the monitor is restarted it can still serve the previous data. These files can also be manually edited and the changes
+will be reloaded in to traffic monitor so that if traffic ops is down or unreachable for an extended period of time manual updates can be done. If on initial startup trafficops is unavailable then traffic monitor will continue through it's
+exponential backoff until it hits the max retry interval, at that point it will create an un-authenticated trafficops session and use the data from disk. It will still poll trafficops for updates though and if it successfully gets through
+then it will login at that point.
+
Formatting Conventions
======================
Go code should be formatted with ``gofmt``. See also ``CONTRIBUTING.md``.
diff --git a/traffic_monitor/config/config.go b/traffic_monitor/config/config.go
index c64d250b1..ba0ee4f96 100644
--- a/traffic_monitor/config/config.go
+++ b/traffic_monitor/config/config.go
@@ -40,6 +40,10 @@ const (
LogLocationNull = "null"
//StaticFileDir is the directory that contains static html and js files.
StaticFileDir = "/opt/traffic_monitor/static/"
+ //CrConfigBackupFile is the default file name to store the last crconfig
+ CRConfigBackupFile = "/opt/traffic_monitor/crconfig.backup"
+ //TmConfigBackupFile is the default file name to store the last tmconfig
+ TMConfigBackupFile = "/opt/traffic_monitor/tmconfig.backup"
)
// Config is the configuration for the application. It includes myriad data, such as polling intervals and log locations.
@@ -67,6 +71,9 @@ type Config struct {
CRConfigHistoryCount uint64 `json:"crconfig_history_count"`
TrafficOpsMinRetryInterval time.Duration `json:"-"`
TrafficOpsMaxRetryInterval time.Duration `json:"-"`
+ CRConfigBackupFile string `json:"crconfig_backup_file"`
+ TMConfigBackupFile string `json:"tmconfig_backup_file"`
+ TrafficOpsDiskRetryMax uint64 `json:"-"`
}
func (c Config) ErrorLog() log.LogLocation { return log.LogLocation(c.LogLocationError) }
@@ -100,6 +107,9 @@ var DefaultConfig = Config{
CRConfigHistoryCount: 20000,
TrafficOpsMinRetryInterval: 100 * time.Millisecond,
TrafficOpsMaxRetryInterval: 60000 * time.Millisecond,
+ CRConfigBackupFile: CRConfigBackupFile,
+ TMConfigBackupFile: TMConfigBackupFile,
+ TrafficOpsDiskRetryMax: 2,
}
// MarshalJSON marshals custom millisecond durations. Aliasing inspired by http://choly.ca/post/go-json-marshalling/
@@ -147,6 +157,9 @@ func (c *Config) UnmarshalJSON(data []byte) error {
ServeWriteTimeoutMs *uint64 `json:"serve_write_timeout_ms"`
TrafficOpsMinRetryIntervalMs *uint64 `json:"traffic_ops_min_retry_interval_ms"`
TrafficOpsMaxRetryIntervalMs *uint64 `json:"traffic_ops_max_retry_interval_ms"`
+ TrafficOpsDiskRetryMax *uint64 `json:"traffic_ops_disk_retry_max"`
+ CRConfigBackupFile *string `json:"crconfig_backup_file"`
+ TMConfigBackupFile *string `json:"tmconfig_backup_file"`
*Alias
}{
Alias: (*Alias)(c),
@@ -192,6 +205,15 @@ func (c *Config) UnmarshalJSON(data []byte) error {
if aux.TrafficOpsMaxRetryIntervalMs != nil {
c.TrafficOpsMaxRetryInterval = time.Duration(*aux.TrafficOpsMaxRetryIntervalMs) * time.Millisecond
}
+ if aux.TrafficOpsDiskRetryMax != nil {
+ c.TrafficOpsDiskRetryMax = *aux.TrafficOpsDiskRetryMax
+ }
+ if aux.CRConfigBackupFile != nil {
+ c.CRConfigBackupFile = *aux.CRConfigBackupFile
+ }
+ if aux.TMConfigBackupFile != nil {
+ c.TMConfigBackupFile = *aux.TMConfigBackupFile
+ }
return nil
}
diff --git a/traffic_monitor/manager/manager.go b/traffic_monitor/manager/manager.go
index 6ba9dc99a..f6f0c0fcd 100644
--- a/traffic_monitor/manager/manager.go
+++ b/traffic_monitor/manager/manager.go
@@ -43,7 +43,7 @@ import (
// Start starts the poller and handler goroutines
//
func Start(opsConfigFile string, cfg config.Config, appData config.StaticAppData, trafficMonitorConfigFileName string) error {
- toSession := towrap.ITrafficOpsSession(towrap.NewTrafficOpsSessionThreadsafe(nil, cfg.CRConfigHistoryCount))
+ toSession := towrap.ITrafficOpsSession(towrap.NewTrafficOpsSessionThreadsafe(nil, cfg.CRConfigHistoryCount, cfg))
localStates := peer.NewCRStatesThreadsafe() // this is the local state as discoverer by this traffic_monitor
fetchCount := threadsafe.NewUint() // note this is the number of individual caches fetched from, not the number of times all the caches were polled.
diff --git a/traffic_monitor/manager/opsconfig.go b/traffic_monitor/manager/opsconfig.go
index 7b34bcd25..6da71bd11 100644
--- a/traffic_monitor/manager/opsconfig.go
+++ b/traffic_monitor/manager/opsconfig.go
@@ -20,9 +20,12 @@ package manager
*/
import (
+ "crypto/tls"
"fmt"
"io/ioutil"
"net"
+ "net/http"
+ "net/http/cookiejar"
"time"
"golang.org/x/sys/unix"
@@ -138,6 +141,7 @@ func StartOpsConfigManager(
trafficOpsRequestTimeout := time.Second * time.Duration(10)
var realToSession *to.Session
var toAddr net.Addr
+ var toLoginCount uint64
// fixed an issue here where traffic_monitor loops forever, doing nothing useful if traffic_ops is down,
// and would never logging in again. since traffic_monitor is just starting up here, keep retrying until traffic_ops is reachable and a session can be established.
@@ -154,6 +158,28 @@ func StartOpsConfigManager(
duration := backoff.BackoffDuration()
log.Errorf("retrying in %v\n", duration)
time.Sleep(duration)
+
+ if toSession.BackupFileExists() && (toLoginCount >= cfg.TrafficOpsDiskRetryMax) {
+ jar, err := cookiejar.New(nil)
+ if err != nil {
+ log.Errorf("Err getting cookiejar")
+ continue
+ }
+
+ realToSession = to.NewSession(newOpsConfig.Username, newOpsConfig.Password, newOpsConfig.Url, staticAppData.UserAgent, &http.Client{
+ Timeout: trafficOpsRequestTimeout,
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ },
+ Jar: jar,
+ }, useCache)
+ toSession.Set(realToSession)
+ // At this point we have a valid 'dummy' session. This will allow us to pull from disk but will also retry when TO comes up
+ log.Errorf("error instantiating Session with traffic_ops, backup disk files exist, creating empty traffic_ops session to read")
+ break
+ }
+
+ toLoginCount++
continue
} else {
toSession.Set(realToSession)
diff --git a/traffic_monitor/towrap/towrap.go b/traffic_monitor/towrap/towrap.go
index a017945b4..d81ba695a 100644
--- a/traffic_monitor/towrap/towrap.go
+++ b/traffic_monitor/towrap/towrap.go
@@ -22,12 +22,15 @@ package towrap
import (
"errors"
"fmt"
+ "io/ioutil"
+ "os"
"strconv"
"sync"
"time"
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/traffic_monitor/config"
"github.com/apache/trafficcontrol/traffic_ops/client"
"github.com/json-iterator/go"
@@ -47,8 +50,11 @@ type ITrafficOpsSession interface {
DeliveryServices() ([]tc.DeliveryService, error)
CacheGroups() ([]tc.CacheGroupNullable, error)
CRConfigHistory() []CRConfigStat
+ BackupFileExists() bool
}
+const localHostIP = "127.0.0.1"
+
var ErrNilSession = fmt.Errorf("nil session")
// TODO rename CRConfigCacheObj
@@ -63,6 +69,15 @@ type ByteMapCache struct {
m *sync.RWMutex
}
+func (s TrafficOpsSessionThreadsafe) BackupFileExists() bool {
+ if _, err := os.Stat(s.CRConfigBackupFile); !os.IsNotExist(err) {
+ if _, err = os.Stat(s.TMConfigBackupFile); !os.IsNotExist(err) {
+ return true
+ }
+ }
+ return false
+}
+
func NewByteMapCache() ByteMapCache {
return ByteMapCache{m: &sync.RWMutex{}, cache: &map[string]ByteTime{}}
}
@@ -148,15 +163,17 @@ type CRConfigStat struct {
// TrafficOpsSessionThreadsafe provides access to the Traffic Ops client safe for multiple goroutines. This fulfills the ITrafficOpsSession interface.
type TrafficOpsSessionThreadsafe struct {
- session **client.Session // pointer-to-pointer, because we're given a pointer from the Traffic Ops package, and we don't want to copy it.
- m *sync.Mutex
- lastCRConfig ByteMapCache
- crConfigHist CRConfigHistoryThreadsafe
+ session **client.Session // pointer-to-pointer, because we're given a pointer from the Traffic Ops package, and we don't want to copy it.
+ m *sync.Mutex
+ lastCRConfig ByteMapCache
+ crConfigHist CRConfigHistoryThreadsafe
+ CRConfigBackupFile string
+ TMConfigBackupFile string
}
// NewTrafficOpsSessionThreadsafe returns a new threadsafe TrafficOpsSessionThreadsafe wrapping the given `Session`.
-func NewTrafficOpsSessionThreadsafe(s *client.Session, crConfigHistoryLimit uint64) TrafficOpsSessionThreadsafe {
- return TrafficOpsSessionThreadsafe{session: &s, m: &sync.Mutex{}, lastCRConfig: NewByteMapCache(), crConfigHist: NewCRConfigHistoryThreadsafe(crConfigHistoryLimit)}
+func NewTrafficOpsSessionThreadsafe(s *client.Session, crConfigHistoryLimit uint64, cfg config.Config) TrafficOpsSessionThreadsafe {
+ return TrafficOpsSessionThreadsafe{session: &s, m: &sync.Mutex{}, lastCRConfig: NewByteMapCache(), crConfigHist: NewCRConfigHistoryThreadsafe(crConfigHistoryLimit), CRConfigBackupFile: cfg.CRConfigBackupFile, TMConfigBackupFile: cfg.TMConfigBackupFile}
}
// Set sets the internal Traffic Ops session. This is safe for multiple goroutines, being aware they will race.
@@ -229,13 +246,35 @@ func (s *TrafficOpsSessionThreadsafe) CRConfigValid(crc *tc.CRConfig, cdn string
// CRConfigRaw returns the CRConfig from the Traffic Ops. This is safe for multiple goroutines.
func (s TrafficOpsSessionThreadsafe) CRConfigRaw(cdn string) ([]byte, error) {
+
ss := s.get()
+
+ var remoteAddr string
+
if ss == nil {
return nil, ErrNilSession
}
+
b, reqInf, err := ss.GetCRConfig(cdn)
+ if err == nil {
+ remoteAddr = reqInf.RemoteAddr.String()
+ ioutil.WriteFile(s.CRConfigBackupFile, b, 0644)
+ } else {
+ if s.BackupFileExists() {
+ b, err = ioutil.ReadFile(s.CRConfigBackupFile)
+ if err != nil {
+ err = errors.New("file Read Error: " + err.Error())
+ return nil, err
+ }
+ remoteAddr = localHostIP
+ log.Errorln("Error getting CRConfig from traffic_ops, backup file exists, reading from file")
+ err = nil
+ } else {
+ return nil, ErrNilSession
+ }
+ }
- hist := &CRConfigStat{time.Now(), reqInf.RemoteAddr.String(), tc.CRConfigStats{}, err}
+ hist := &CRConfigStat{time.Now(), remoteAddr, tc.CRConfigStats{}, err}
defer s.crConfigHist.Add(hist)
if err != nil {
@@ -277,8 +316,33 @@ func (s TrafficOpsSessionThreadsafe) trafficMonitorConfigMapRaw(cdn string) (*tc
if ss == nil {
return nil, ErrNilSession
}
- configMap, _, error := ss.GetTrafficMonitorConfigMap(cdn)
- return configMap, error
+
+ configMap, _, err := ss.GetTrafficMonitorConfigMap(cdn)
+ if err != nil {
+ // Default error case, no backup file exists
+ if !s.BackupFileExists() {
+ return configMap, err
+ }
+
+ b, err := ioutil.ReadFile(s.TMConfigBackupFile)
+ if err != nil {
+ return nil, errors.New("reading TMConfigBackupFile: " + err.Error())
+ }
+
+ log.Errorln("Error getting configMap from traffic_ops, backup file exists, reading from file")
+ json := jsoniter.ConfigFastest
+ if err := json.Unmarshal(b, &configMap); err != nil {
+ return nil, errors.New("unmarhsalling backup file monitoring.json: " + err.Error())
+ }
+ return configMap, err
+ }
+
+ json := jsoniter.ConfigFastest
+ data, err := json.Marshal(*configMap)
+ if err == nil {
+ ioutil.WriteFile(s.TMConfigBackupFile, data, 0644)
+ }
+ return configMap, err
}
// TrafficMonitorConfigMap returns the Traffic Monitor config map from the Traffic Ops. This is safe for multiple goroutines.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services