You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2023/01/05 16:59:41 UTC

[beam] branch master updated: Reduce lock contention for metrics hashName hasher (#24881)

This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new c49512298fa Reduce lock contention for metrics hashName hasher (#24881)
c49512298fa is described below

commit c49512298fa0cbc08456dcf02ad9228773e868a4
Author: Shane Hansen <sh...@gmail.com>
AuthorDate: Thu Jan 5 08:59:32 2023 -0800

    Reduce lock contention for metrics hashName hasher (#24881)
    
    * Add metrics parallel bench
    
    Utilize parallel benchmark for metrics to measure scalability on
    multiple cores.
    
    * Reduce locking for hashName
    
    Use a pool of hashers rather than a singleton. For workloads
    that create Counters on the fly this results in a 4x or so overall speedup in
    adding incrementing Counters.
---
 sdks/go/pkg/beam/core/metrics/metrics.go      | 22 ++++++++++------
 sdks/go/pkg/beam/core/metrics/metrics_test.go | 38 +++++++++++++++------------
 2 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/sdks/go/pkg/beam/core/metrics/metrics.go b/sdks/go/pkg/beam/core/metrics/metrics.go
index 678a34a647d..93dd9d0070e 100644
--- a/sdks/go/pkg/beam/core/metrics/metrics.go
+++ b/sdks/go/pkg/beam/core/metrics/metrics.go
@@ -49,6 +49,7 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"hash"
 	"hash/fnv"
 	"sort"
 	"sync"
@@ -221,28 +222,33 @@ func newName(ns, n string) name {
 // We hash the name to a uint64 so we avoid using go's native string hashing for
 // every use of a metrics. uint64s have faster lookup than strings as a result.
 // Collisions are possible, but statistically unlikely as namespaces and names
-// are usually short enough to avoid this.
+// are usually short enough to avoid this. A sync.Pool is used  because it can provide
+// goroutine-local values that reduce contention and profiling shows hashName from NewCounter
+// can be a contention hotspot. See parallel benches metrics_test.go:BenchmarkMetrics/*
 var (
-	hasherMu sync.Mutex
-	hasher   = fnv.New64a()
+	hashPool = sync.Pool{
+		New: func() interface{} {
+			return fnv.New64a()
+		},
+	}
 )
 
 func hashName(ns, n string) nameHash {
-	hasherMu.Lock()
+	hasher := hashPool.Get().(hash.Hash64)
 	hasher.Reset()
 	var buf [64]byte
 	b := buf[:]
-	hashString(ns, b)
-	hashString(n, b)
+	hashString(hasher, ns, b)
+	hashString(hasher, n, b)
 	h := hasher.Sum64()
-	hasherMu.Unlock()
+	hashPool.Put(hasher)
 	return nameHash(h)
 }
 
 // hashString hashes a string with the package level hasher
 // and requires posession of the hasherMu lock. The byte
 // slice is assumed to be backed by a [64]byte.
-func hashString(s string, b []byte) {
+func hashString(hasher hash.Hash64, s string, b []byte) {
 	l := len(s)
 	i := 0
 	for len(s)-i > 64 {
diff --git a/sdks/go/pkg/beam/core/metrics/metrics_test.go b/sdks/go/pkg/beam/core/metrics/metrics_test.go
index 75b483184ab..33d1b2c7ddd 100644
--- a/sdks/go/pkg/beam/core/metrics/metrics_test.go
+++ b/sdks/go/pkg/beam/core/metrics/metrics_test.go
@@ -617,21 +617,23 @@ func TestPcolQueryResult(t *testing.T) {
 	}
 }
 
-// Run on @lostluck's desktop (2020/01/21) go1.13.4
+// Run on @shanemhansen's desktop (2022/01/03) go1.20 RC1 after changing hashName to use a pool of hashers
+// sync.Pool can return thread-local results eliminating the need for a lock and increasing throughput.
+// There are users in the wild who create an excessive number of Counters so a 4x improvement in throughput at the expense of
+// creating GOMAXPROCS hasher values seems reasonable.
 //
-// Allocs & bytes should be consistent within go versions, but ns/op is relative to the running machine.
-//
-// BenchmarkMetrics/counter_inplace-12              6054129               208 ns/op              48 B/op          1 allocs/op
-// BenchmarkMetrics/distribution_inplace-12         5707147               228 ns/op              48 B/op          1 allocs/op
-// BenchmarkMetrics/gauge_inplace-12                4742331               259 ns/op              48 B/op          1 allocs/op
-// BenchmarkMetrics/counter_predeclared-12         90147133                12.7 ns/op             0 B/op          0 allocs/op
-// BenchmarkMetrics/distribution_predeclared-12            55396678                21.6 ns/op             0 B/op          0 allocs/op
-// BenchmarkMetrics/gauge_predeclared-12                   18535839                60.5 ns/op             0 B/op          0 allocs/op
-// BenchmarkMetrics/counter_raw-12                         159581343                7.18 ns/op            0 B/op          0 allocs/op
-// BenchmarkMetrics/distribution_raw-12                    82724314                14.7 ns/op             0 B/op          0 allocs/op
-// BenchmarkMetrics/gauge_raw-12                           23292386                55.2 ns/op             0 B/op          0 allocs/op
-// BenchmarkMetrics/getStore-12                            309361303                3.78 ns/op            0 B/op          0 allocs/op
-// BenchmarkMetrics/getCounterSet-12                       287720998                3.98 ns/op            0 B/op          0 allocs/op
+// name                                 old time/op  new time/op  delta
+// Metrics/counter_inplace-12            376ns ±17%    88ns ± 7%  -76.66%  (p=0.008 n=5+5)
+// Metrics/distribution_inplace-12       394ns ± 3%   153ns ± 8%  -61.17%  (p=0.008 n=5+5)
+// Metrics/gauge_inplace-12              371ns ± 4%   258ns ± 1%  -30.37%  (p=0.008 n=5+5)
+// Metrics/counter_predeclared-12       16.9ns ± 6%  17.0ns ± 3%     ~     (p=0.595 n=5+5)
+// Metrics/distribution_predeclared-12  83.2ns ± 2%  84.9ns ± 1%     ~     (p=0.056 n=5+5)
+// Metrics/gauge_predeclared-12          105ns ± 6%   110ns ± 5%   +4.81%  (p=0.032 n=5+5)
+// Metrics/counter_raw-12               10.8ns ± 4%  12.0ns ±28%     ~     (p=0.151 n=5+5)
+// Metrics/distribution_raw-12          77.6ns ± 7%  78.8ns ± 5%     ~     (p=0.841 n=5+5)
+// Metrics/gauge_raw-12                 78.9ns ± 1%  77.3ns ± 4%     ~     (p=0.151 n=5+5)
+// Metrics/getStore-12                  0.27ns ± 3%  0.27ns ± 2%     ~     (p=0.841 n=5+5)
+// Metrics/getCounterSet-12             0.32ns ± 3%  0.31ns ± 0%   -1.28%  (p=0.048 n=5+4)
 func BenchmarkMetrics(b *testing.B) {
 	pt, c, d, g := "bench.bundle.data", "counter", "distribution", "gauge"
 	aBundleID := "benchBID"
@@ -664,9 +666,11 @@ func BenchmarkMetrics(b *testing.B) {
 	}
 	for _, test := range tests {
 		b.Run(test.name, func(b *testing.B) {
-			for i := 0; i < b.N; i++ {
-				test.call()
-			}
+			b.RunParallel(func(pb *testing.PB) {
+				for pb.Next() {
+					test.call()
+				}
+			})
 		})
 	}
 }