You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by de...@apache.org on 2018/07/05 20:10:51 UTC
[trafficcontrol] 01/05: add cpu profiling process,
and memory profiling and db stats endpoints to traffic_ops_golang
This is an automated email from the ASF dual-hosted git repository.
dewrich pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
commit deb208f0a73c1a62fae38d1fd674d6caaebc959c
Author: Dylan Volz <Dy...@comcast.com>
AuthorDate: Mon Jul 2 10:18:54 2018 -0600
add cpu profiling process, and memory profiling and db stats endpoints to traffic_ops_golang
---
traffic_ops/app/conf/cdn.conf | 3 +-
traffic_ops/traffic_ops_golang/config/config.go | 24 ++++--
traffic_ops/traffic_ops_golang/routes.go | 50 ++++++++++++
traffic_ops/traffic_ops_golang/routing.go | 1 +
.../traffic_ops_golang/traffic_ops_golang.go | 91 +++++++++++++++++++++-
5 files changed, 159 insertions(+), 10 deletions(-)
diff --git a/traffic_ops/app/conf/cdn.conf b/traffic_ops/app/conf/cdn.conf
index 7f1a13f..d365c3a 100644
--- a/traffic_ops/app/conf/cdn.conf
+++ b/traffic_ops/app/conf/cdn.conf
@@ -27,7 +27,8 @@
"max_db_connections": 20,
"backend_max_connections": {
"mojolicious": 4
- }
+ },
+ "profiling_enabled": false
},
"cors" : {
"access_control_allow_origin" : "*"
diff --git a/traffic_ops/traffic_ops_golang/config/config.go b/traffic_ops/traffic_ops_golang/config/config.go
index b47d402..c90c4e1 100644
--- a/traffic_ops/traffic_ops_golang/config/config.go
+++ b/traffic_ops/traffic_ops_golang/config/config.go
@@ -75,6 +75,8 @@ type ConfigTrafficOpsGolang struct {
Insecure bool `json:"insecure"`
MaxDBConnections int `json:"max_db_connections"`
BackendMaxConnections map[string]int `json:"backend_max_connections"`
+ ProfilingEnabled bool `json:"profiling_enabled"`
+ ProfilingLocation string `json:"profiling_location"`
}
// ConfigDatabase reflects the structure of the database.conf file
@@ -127,20 +129,32 @@ func (c Config) EventLog() log.LogLocation {
const BlockStartup = true
const AllowStartup = false
-// LoadConfig - reads the config file into the Config struct
-func LoadConfig(cdnConfPath string, dbConfPath string, riakConfPath string, appVersion string) (Config, []error, bool) {
+
+func LoadCdnConfig(cdnConfPath string) (Config, error) {
// load json from cdn.conf
confBytes, err := ioutil.ReadFile(cdnConfPath)
if err != nil {
- return Config{}, []error{fmt.Errorf("reading CDN conf '%s': %v", cdnConfPath, err)}, BlockStartup
+ return Config{}, fmt.Errorf("reading CDN conf '%s': %v", cdnConfPath, err)
}
- cfg := Config{Version: appVersion}
+ cfg := Config{}
err = json.Unmarshal(confBytes, &cfg)
if err != nil {
- return Config{}, []error{fmt.Errorf("unmarshalling '%s': %v", cdnConfPath, err)}, BlockStartup
+ return Config{}, fmt.Errorf("unmarshalling '%s': %v", cdnConfPath, err)
+ }
+ return cfg, nil
+}
+
+// LoadConfig - reads the config file into the Config struct
+
+func LoadConfig(cdnConfPath string, dbConfPath string, riakConfPath string, appVersion string) (Config, []error, bool) {
+ // load cdn.conf
+ cfg, err := LoadCdnConfig(cdnConfPath)
+ if err != nil {
+ return Config{}, []error{fmt.Errorf("Loading cdn config from '%s': %v", cdnConfPath, err)}, BlockStartup
}
+ cfg.Version = appVersion
// load json from database.conf
dbConfBytes, err := ioutil.ReadFile(dbConfPath)
diff --git a/traffic_ops/traffic_ops_golang/routes.go b/traffic_ops/traffic_ops_golang/routes.go
index dbff573..33acff1 100644
--- a/traffic_ops/traffic_ops_golang/routes.go
+++ b/traffic_ops/traffic_ops_golang/routes.go
@@ -21,11 +21,14 @@ package main
import (
"crypto/tls"
+ "encoding/json"
+ "errors"
"io"
"log"
"net"
"net/http"
"net/http/httputil"
+ "runtime"
"time"
tclog "github.com/apache/trafficcontrol/lib/go-log"
@@ -61,6 +64,7 @@ import (
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/types"
"github.com/basho/riak-go-client"
+ "github.com/jmoiron/sqlx"
)
// Authenticated ...
@@ -354,11 +358,57 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) {
{http.MethodGet, `tools/write_crconfig/{cdn}/?$`, crconfig.SnapshotOldGUIHandler(d.DB, d.Config), auth.PrivLevelOperations, Authenticated, nil},
// DEPRECATED - use GET /api/1.2/cdns/{cdn}/snapshot
{http.MethodGet, `CRConfig-Snapshots/{cdn}/CRConfig.json?$`, crconfig.SnapshotOldGetHandler(d.DB, d.Config), auth.PrivLevelReadOnly, Authenticated, nil},
+ // USED FOR Debugging
+ {http.MethodGet, `admin/memory-stats`, memoryStatsHandler(d.Profiling), auth.PrivLevelOperations, Authenticated, nil},
+ {http.MethodGet, `admin/db-stats`, dbStatsHandler(d.Profiling, d.DB), auth.PrivLevelOperations, Authenticated, nil},
}
return routes, rawRoutes, proxyHandler, nil
}
+func memoryStatsHandler(profiling *bool) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ handleErrs := tc.GetHandleErrorsFunc(w, r)
+ if *profiling {
+ stats := runtime.MemStats{}
+ runtime.ReadMemStats(&stats)
+
+ bytes, err := json.Marshal(stats)
+ if err != nil {
+ tclog.Errorln("unable to marshal stats: " + err.Error())
+ handleErrs(http.StatusInternalServerError, errors.New("marshalling error"))
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Write(bytes)
+ } else {
+ handleErrs(http.StatusPreconditionFailed, errors.New("profiling is not enabled"))
+ return
+ }
+ }
+}
+
+func dbStatsHandler(profiling *bool, db *sqlx.DB) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ handleErrs := tc.GetHandleErrorsFunc(w, r)
+ if *profiling {
+ stats := db.DB.Stats()
+
+ bytes, err := json.Marshal(stats)
+ if err != nil {
+ tclog.Errorln("unable to marshal stats: " + err.Error())
+ handleErrs(http.StatusInternalServerError, errors.New("marshalling error"))
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Write(bytes)
+ } else {
+ handleErrs(http.StatusPreconditionFailed, errors.New("profiling is not enabled"))
+ return
+ }
+ }
+}
+
// RootHandler returns the / handler for the service, which reverse-proxies the old Perl Traffic Ops
func rootHandler(d ServerData) http.Handler {
tr := &http.Transport{
diff --git a/traffic_ops/traffic_ops_golang/routing.go b/traffic_ops/traffic_ops_golang/routing.go
index 24abbdb..c675aa7 100644
--- a/traffic_ops/traffic_ops_golang/routing.go
+++ b/traffic_ops/traffic_ops_golang/routing.go
@@ -74,6 +74,7 @@ func getDefaultMiddleware(secret string) []Middleware {
type ServerData struct {
config.Config
DB *sqlx.DB
+ Profiling *bool // Yes this is a field in the config but we want to live reload this value and NOT the entire config
}
// CompiledRoute ...
diff --git a/traffic_ops/traffic_ops_golang/traffic_ops_golang.go b/traffic_ops/traffic_ops_golang/traffic_ops_golang.go
index f5e2865..15644d3 100644
--- a/traffic_ops/traffic_ops_golang/traffic_ops_golang.go
+++ b/traffic_ops/traffic_ops_golang/traffic_ops_golang.go
@@ -25,14 +25,19 @@ import (
"fmt"
"net/http"
"os"
+ "runtime/pprof"
"time"
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/about"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/config"
+ "os/signal"
+ "path/filepath"
+
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
+ "golang.org/x/sys/unix"
)
// set the version at build time: `go build -X "main.version=..."`
@@ -117,7 +122,9 @@ func main() {
db.SetMaxOpenConns(cfg.MaxDBConnections)
- if err := RegisterRoutes(ServerData{DB: db, Config: cfg}); err != nil {
+ profiling := cfg.ProfilingEnabled
+
+ if err := RegisterRoutes(ServerData{DB: db, Config: cfg, Profiling: &profiling}); err != nil {
log.Errorf("registering routes: %v\n", err)
return
}
@@ -134,8 +141,84 @@ func main() {
ErrorLog: log.Error,
}
- if err := server.ListenAndServeTLS(cfg.CertPath, cfg.KeyPath); err != nil {
- log.Errorf("stopping server: %v\n", err)
- return
+ go func() {
+ if err := server.ListenAndServeTLS(cfg.CertPath, cfg.KeyPath); err != nil {
+ log.Errorf("stopping server: %v\n", err)
+ panic(err)
+ }
+ }()
+
+ profilingLocation := os.TempDir()
+
+ if cfg.LogLocationError != "" && cfg.LogLocationError != "stdout" {
+ errorDir := filepath.Dir(cfg.LogLocationError)
+
+ if _, err := os.Stat(errorDir); err == nil {
+ profilingLocation = errorDir
+ }
+ }
+
+ profilingLocation = filepath.Join(profilingLocation, "profiling")
+ if cfg.ProfilingLocation != "" {
+ profilingLocation = cfg.ProfilingLocation
+ } else {
+ //if it isn't a provided location create the profiling directory under the default temp location
+ err = os.Mkdir(profilingLocation, 0755)
+ if err != nil {
+ log.Errorf("unable to create profiling location: %s\n", err.Error())
+ }
+ }
+
+ reloadProfilingConfig := func() {
+ log.Debugln("received SIGHUP")
+ newCfg, err := config.LoadCdnConfig(*configFileName)
+ if err != nil {
+ log.Errorln("reloading config: ", err.Error())
+ }
+ profiling = newCfg.ProfilingEnabled
+ if newCfg.ProfilingLocation != "" {
+ profilingLocation = cfg.ProfilingLocation
+ log.Infof("profiling location: %s\n", profilingLocation)
+ }
+ if profiling {
+ log.Infoln("profiling enabled")
+ }
+ }
+
+ log.Infof("profiling location: %s\n", profilingLocation)
+ if profiling {
+ log.Infoln("profiling enabled")
+ }
+ continuousProfile(&profiling, &profilingLocation, cfg.Version)
+
+ signalReloader(unix.SIGHUP, reloadProfilingConfig)
+}
+
+func continuousProfile(profiling *bool, profilingDir *string, version string) {
+ go func() {
+ for {
+ if *profiling {
+ now := time.Now().UTC()
+ filename := filepath.Join(*profilingDir, fmt.Sprintf("tocpu-%s-%s.pprof", version, now.Format(time.RFC3339)))
+ f, err := os.Create(filename)
+ if err != nil {
+ log.Errorf("creating profile: %v\n", err)
+ os.Exit(1)
+ }
+
+ pprof.StartCPUProfile(f)
+ time.Sleep(time.Minute)
+ pprof.StopCPUProfile()
+ f.Close()
+ }
+ }
+ }()
+}
+
+func signalReloader(sig os.Signal, f func()) {
+ c := make(chan os.Signal, 1)
+ signal.Notify(c, sig)
+ for range c {
+ f()
}
}