You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ne...@apache.org on 2016/12/08 18:46:15 UTC
[13/20] incubator-trafficcontrol git commit: Fix TM2 poller to use
nanosleep, not busywait
Fix TM2 poller to use nanosleep, not busywait
Fixes Traffic Monitor 2.0 to use nanosleep instead of busywaiting
on Linux builds for polling (recalling we can't Sleep() because it
gets progressively slower, in what appears a Go runtime bug).
Changes GOMAXPROCS to 3x the number of cores, to steal from other
processes, plus 3 for the 3 thread-locked pollers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/c3ab47dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/c3ab47dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/c3ab47dc
Branch: refs/heads/master
Commit: c3ab47dccf41fd04b7638dd0eacad3c7ed8cdd99
Parents: 9153d3e
Author: Robert Butts <ro...@gmail.com>
Authored: Wed Dec 7 13:51:03 2016 -0700
Committer: Dave Neuman <ne...@apache.org>
Committed: Thu Dec 8 11:44:33 2016 -0700
----------------------------------------------------------------------
.../experimental/common/poller/poller.go | 37 ++++++++++----------
.../experimental/common/poller/threadsleep.go | 17 +++++++++
.../common/poller/threadsleep_linux.go | 23 ++++++++++++
.../traffic_monitor/traffic_monitor.go | 10 +++---
4 files changed, 65 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/c3ab47dc/traffic_monitor/experimental/common/poller/poller.go
----------------------------------------------------------------------
diff --git a/traffic_monitor/experimental/common/poller/poller.go b/traffic_monitor/experimental/common/poller/poller.go
index 7c86b57..68d15ce 100644
--- a/traffic_monitor/experimental/common/poller/poller.go
+++ b/traffic_monitor/experimental/common/poller/poller.go
@@ -25,6 +25,7 @@ import (
"math/rand"
"net/http"
"os"
+ "runtime"
"sync/atomic"
"time"
@@ -192,6 +193,15 @@ func (p HttpPoller) SleepPoll() {
}
}
+func mustDie(die <-chan struct{}) bool {
+ select {
+ case <-die:
+ return true
+ default:
+ }
+ return false
+}
+
// TODO iterationCount and/or p.TickChan?
func sleepPoller(interval time.Duration, id string, url string, fetcher fetcher.Fetcher, die <-chan struct{}) {
pollSpread := time.Duration(rand.Float64()*float64(interval/time.Nanosecond)) * time.Nanosecond
@@ -200,10 +210,8 @@ func sleepPoller(interval time.Duration, id string, url string, fetcher fetcher.
lastTime := time.Now()
for {
select {
- case now := <-tick.C:
- tick.Stop() // old ticker MUST call Stop() to release resources. Else, memory leak.
- tick = time.NewTicker(interval) // recreate timer, to avoid Go's "smoothing" nonsense
- realInterval := now.Sub(lastTime)
+ case <-tick.C:
+ realInterval := time.Now().Sub(lastTime)
if realInterval > interval+(time.Millisecond*100) {
instr.TimerFail.Inc()
log.Infof("Intended Duration: %v Actual Duration: %v\n", interval, realInterval)
@@ -259,6 +267,7 @@ func (p HttpPoller) InsomniacPoll() {
}
func insomniacPoller(pollerId int64, polls []HTTPPollInfo, fetcherTemplate fetcher.HttpFetcher, die <-chan struct{}) {
+ runtime.LockOSThread()
fmt.Printf("httpPoll %v called\n", pollerId)
heap := Heap{PollerID: pollerId}
start := time.Now()
@@ -277,14 +286,6 @@ func insomniacPoller(pollerId int64, polls []HTTPPollInfo, fetcherTemplate fetch
fetchers[p.ID] = fetcher
}
fmt.Printf("httpPoll %v added to heap\n", pollerId)
- mustDie := func() bool {
- select {
- case <-die:
- return true
- default:
- }
- return false
- }
timeMax := func(a time.Time, b time.Time) time.Time {
if a.After(b) {
@@ -311,21 +312,21 @@ func insomniacPoller(pollerId int64, polls []HTTPPollInfo, fetcherTemplate fetch
fmt.Printf("httpPoll %v starting main loop\n", pollerId)
for {
- if mustDie() {
+ if mustDie(die) {
fmt.Printf("httpPoll %v dying\n", pollerId)
return
}
p, ok := heap.Pop()
if !ok {
- fmt.Printf("httpPoll %v empty heap, busylooping\n", pollerId)
- continue //busywait because we fear to sleep. TODO sleep?
+ ThreadSleep(0)
+ continue
}
if p.Info.ID == "odol-atsec-jac-04" {
fmt.Printf("httpPoll %v popped id %v p.Next %v now %v\n", pollerId, p.Info.ID, p.Next, time.Now())
}
- for p.Next.After(time.Now()) {
- // busywait, because sleeping gets progressively slower for unknown reasons.
- }
+
+ ThreadSleep(p.Next.Sub(time.Now()))
+
if p.Info.ID == "odol-atsec-jac-04" {
fmt.Printf("httpPoll %v polling %v next %v now %v\n", pollerId, p.Info.ID, p.Next, time.Now())
}
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/c3ab47dc/traffic_monitor/experimental/common/poller/threadsleep.go
----------------------------------------------------------------------
diff --git a/traffic_monitor/experimental/common/poller/threadsleep.go b/traffic_monitor/experimental/common/poller/threadsleep.go
new file mode 100644
index 0000000..b8b1233
--- /dev/null
+++ b/traffic_monitor/experimental/common/poller/threadsleep.go
@@ -0,0 +1,17 @@
+// +build !linux
+
+package poller
+
+import (
+ "runtime"
+ "time"
+)
+
+// ThreadSleep actually busywaits for the given duration. This is becuase Go doesn't have Mac and Windows nanosleep syscalls, and `Sleep` sleeps for progressively longer than requested.
+func ThreadSleep(d time.Duration) {
+ // TODO fix to not busywait on Mac, Windows. We can't simply Sleep, because Sleep gets progressively slower as the app runs, due to a Go runtime issue. If this is changed, you MUST verify the poll doesn't get slower after the app runs for several days.
+ end := time.Now().Add(d)
+ for end.After(time.Now()) {
+ runtime.Gosched()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/c3ab47dc/traffic_monitor/experimental/common/poller/threadsleep_linux.go
----------------------------------------------------------------------
diff --git a/traffic_monitor/experimental/common/poller/threadsleep_linux.go b/traffic_monitor/experimental/common/poller/threadsleep_linux.go
new file mode 100644
index 0000000..9037027
--- /dev/null
+++ b/traffic_monitor/experimental/common/poller/threadsleep_linux.go
@@ -0,0 +1,23 @@
+// +build linux
+
+package poller
+
+import (
+ "errors"
+ "golang.org/x/sys/unix"
+ "time"
+)
+
+// ThreadSleep sleeps using the POSIX syscall `nanosleep`. Note this does not sleep the goroutine, but the operating system thread itself. This should only be called by a goroutine which has previously called `LockOSThread`. This exists due to a bug with `time.Sleep` getting progressively slower as the app runs, and should be removed if the bug in Go is fixed.
+func ThreadSleep(d time.Duration) {
+ if d < 0 {
+ d = 0
+ }
+ t := unix.Timespec{}
+ leftover := unix.NsecToTimespec(d.Nanoseconds())
+ err := errors.New("")
+ for err != nil && (leftover.Sec != 0 || leftover.Nsec != 0) {
+ t = leftover
+ err = unix.Nanosleep(&t, &leftover)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/c3ab47dc/traffic_monitor/experimental/traffic_monitor/traffic_monitor.go
----------------------------------------------------------------------
diff --git a/traffic_monitor/experimental/traffic_monitor/traffic_monitor.go b/traffic_monitor/experimental/traffic_monitor/traffic_monitor.go
index 1b726d2..f2b5444 100644
--- a/traffic_monitor/experimental/traffic_monitor/traffic_monitor.go
+++ b/traffic_monitor/experimental/traffic_monitor/traffic_monitor.go
@@ -8,9 +8,9 @@ package main
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,6 @@ package main
* under the License.
*/
-
import (
"bytes"
"flag"
@@ -116,8 +115,11 @@ func getLogWriters(errLoc, warnLoc, infoLoc, debugLoc string) (io.Writer, io.Wri
return errW, warnW, infoW, debugW, nil
}
+// NumLockedThreads is the number of threads which will be locked to a single goroutine. Ideally this would be computed, but there's no easy way to do that, and it isn't critical. This only protects against the case where a user has a single core, or manually changes GOMAXPROCS to a small number.
+const NumLockedThreads = 3
+
func main() {
- runtime.GOMAXPROCS(runtime.NumCPU())
+ runtime.GOMAXPROCS(runtime.NumCPU()*3 + NumLockedThreads)
staticData, err := getStaticAppData()
if err != nil {