You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by as...@apache.org on 2021/03/12 15:39:35 UTC
[camel-k] 04/10: test: Add reconciliation duration metric e2e test
This is an automated email from the ASF dual-hosted git repository.
astefanutti pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 96adbc75c2549e0e433d4a377c004f0d2247b803
Author: Antonin Stefanutti <an...@stefanutti.fr>
AuthorDate: Wed Mar 10 14:26:53 2021 +0100
test: Add reconciliation duration metric e2e test
---
e2e/common/metrics_test.go | 275 +++++++++++++++++++++++++++++++++-------
e2e/support/util/log_counter.go | 47 +++++++
2 files changed, 276 insertions(+), 46 deletions(-)
diff --git a/e2e/common/metrics_test.go b/e2e/common/metrics_test.go
index 185045e..d4ce252 100644
--- a/e2e/common/metrics_test.go
+++ b/e2e/common/metrics_test.go
@@ -29,7 +29,8 @@ import (
"time"
. "github.com/onsi/gomega"
- "github.com/onsi/gomega/gstruct"
+ . "github.com/onsi/gomega/gstruct"
+ "github.com/onsi/gomega/types"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
@@ -78,22 +79,22 @@ func TestMetrics(t *testing.T) {
// Check it's consistent with the duration observed from logs
var ts1, ts2 time.Time
err = NewLogWalker(&logs).
- AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ AddStep(MatchFields(IgnoreExtras, Fields{
"LoggerName": Equal("camel-k.controller.build"),
"Message": Equal("Build state transition"),
"Phase": Equal("Pending"),
"RequestName": Equal(build.Name),
}), LogEntryNoop).
- AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ AddStep(MatchFields(IgnoreExtras, Fields{
"LoggerName": Equal("camel-k.controller.build"),
"Message": Equal("Reconciling Build"),
"RequestName": Equal(build.Name),
}), func(l *LogEntry) { ts1 = l.Timestamp.Time }).
- AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ AddStep(MatchFields(IgnoreExtras, Fields{
"LoggerName": Equal("camel-k.builder"),
"Message": HavePrefix("resolved image"),
}), LogEntryNoop).
- AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ AddStep(MatchFields(IgnoreExtras, Fields{
"LoggerName": Equal("camel-k.controller.build"),
"Message": Equal("Reconciling Build"),
"RequestName": Equal(build.Name),
@@ -108,60 +109,223 @@ func TestMetrics(t *testing.T) {
Expect(math.Abs((durationFromLogs - duration).Seconds())).To(BeNumerically("<", 1))
// Check the duration is observed in the corresponding metric
- Expect(metrics).To(HaveKeyWithValue("camel_k_build_duration_seconds", gstruct.PointTo(Equal(prometheus.MetricFamily{
- Name: stringP("camel_k_build_duration_seconds"),
- Help: stringP("Camel K build duration"),
- Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
- Metric: []*prometheus.Metric{
- {
- Label: []*prometheus.LabelPair{
- label("result", "Succeeded"),
- },
- Histogram: &prometheus.Histogram{
- SampleCount: uint64P(1),
- SampleSum: float64P(duration.Seconds()),
- Bucket: buckets(duration.Seconds(), []float64{30, 60, 90, 120, 300, 600, math.Inf(1)}),
+ Expect(metrics).To(HaveKeyWithValue("camel_k_build_duration_seconds",
+ EqualP(prometheus.MetricFamily{
+ Name: stringP("camel_k_build_duration_seconds"),
+ Help: stringP("Camel K build duration"),
+ Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
+ Metric: []*prometheus.Metric{
+ {
+ Label: []*prometheus.LabelPair{
+ label("result", "Succeeded"),
+ },
+ Histogram: &prometheus.Histogram{
+ SampleCount: uint64P(1),
+ SampleSum: float64P(duration.Seconds()),
+ Bucket: buckets(duration.Seconds(), []float64{30, 60, 90, 120, 300, 600, math.Inf(1)}),
+ },
},
},
- },
- }))))
+ }),
+ ))
})
- t.Run("Build recovery attempts", func(t *testing.T) {
+ t.Run("Build recovery attempts metric", func(t *testing.T) {
// Check there are no failures reported in the Build status
Expect(build.Status.Failure).To(BeNil())
// Check no recovery attempts are reported in the logs
- recoveryAttemptLogged := false
- err = NewLogWalker(&logs).
- AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
- "LoggerName": Equal("camel-k.controller.build"),
- "Message": HavePrefix("Recovery attempt"),
- "Kind": Equal("Build"),
- "RequestName": Equal(build.Name),
- }), func(l *LogEntry) { recoveryAttemptLogged = true }).
- Walk()
+ recoveryAttempts, err := NewLogCounter(&logs).Count(MatchFields(IgnoreExtras, Fields{
+ "LoggerName": Equal("camel-k.controller.build"),
+ "Message": HavePrefix("Recovery attempt"),
+ "Kind": Equal("Build"),
+ "RequestName": Equal(build.Name),
+ }))
Expect(err).To(BeNil())
- Expect(recoveryAttemptLogged).To(BeFalse())
+ Expect(recoveryAttempts).To(BeNumerically("==", 0))
// Check no recovery attempts are observed in the corresponding metric
- Expect(metrics).To(HaveKeyWithValue("camel_k_build_recovery_attempts", gstruct.PointTo(Equal(prometheus.MetricFamily{
- Name: stringP("camel_k_build_recovery_attempts"),
- Help: stringP("Camel K build recovery attempts"),
- Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
- Metric: []*prometheus.Metric{
- {
- Label: []*prometheus.LabelPair{
- label("result", "Succeeded"),
- },
- Histogram: &prometheus.Histogram{
- SampleCount: uint64P(1),
- SampleSum: float64P(0),
- Bucket: buckets(0, []float64{0, 1, 2, 3, 4, 5, math.Inf(1)}),
+ Expect(metrics).To(HaveKeyWithValue("camel_k_build_recovery_attempts",
+ EqualP(prometheus.MetricFamily{
+ Name: stringP("camel_k_build_recovery_attempts"),
+ Help: stringP("Camel K build recovery attempts"),
+ Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
+ Metric: []*prometheus.Metric{
+ {
+ Label: []*prometheus.LabelPair{
+ label("result", "Succeeded"),
+ },
+ Histogram: &prometheus.Histogram{
+ SampleCount: uint64P(1),
+ SampleSum: float64P(0),
+ Bucket: buckets(0, []float64{0, 1, 2, 3, 4, 5, math.Inf(1)}),
+ },
},
},
- },
- }))))
+ }),
+ ))
+ })
+
+ t.Run("reconciliation duration metric", func(t *testing.T) {
+ Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
+ PointTo(MatchFields(IgnoreExtras, Fields{
+ "Name": EqualP("camel_k_reconciliation_duration_seconds"),
+ "Help": EqualP("Camel K reconciliation loop duration"),
+ "Type": EqualP(prometheus.MetricType_HISTOGRAM),
+ })),
+ ))
+
+ counter := NewLogCounter(&logs)
+
+ // Count the number of IntegrationPlatform reconciliations
+ platformReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
+ "LoggerName": Equal("camel-k.controller.integrationplatform"),
+ "Message": Equal("Reconciling IntegrationPlatform"),
+ "RequestNamespace": Equal(ns),
+ "RequestName": Equal("camel-k"),
+ }))
+ Expect(err).To(BeNil())
+
+ // Check it matches the observation in the corresponding metric
+ platformReconciled := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
+ MatchFieldsP(IgnoreExtras, Fields{
+ "Label": ConsistOf(
+ label("group", camelv1.SchemeGroupVersion.Group),
+ label("version", camelv1.SchemeGroupVersion.Version),
+ label("kind", "IntegrationPlatform"),
+ label("namespace", ns),
+ label("result", "Reconciled"),
+ label("tag", ""),
+ ),
+ }))
+ Expect(platformReconciled).NotTo(BeNil())
+ platformReconciledCount := *platformReconciled.Histogram.SampleCount
+ Expect(platformReconciledCount).To(BeNumerically(">", 0))
+
+ platformRequeued := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
+ MatchFieldsP(IgnoreExtras, Fields{
+ "Label": ConsistOf(
+ label("group", camelv1.SchemeGroupVersion.Group),
+ label("version", camelv1.SchemeGroupVersion.Version),
+ label("kind", "IntegrationPlatform"),
+ label("namespace", ns),
+ label("result", "Requeued"),
+ label("tag", ""),
+ ),
+ }))
+ platformRequeuedCount := uint64(0)
+ if platformRequeued != nil {
+ platformRequeuedCount = *platformRequeued.Histogram.SampleCount
+ }
+
+ Expect(platformReconciliations).To(BeNumerically("==", platformReconciledCount+platformRequeuedCount))
+
+ // Count the number of Integration reconciliations
+ integrationReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
+ "LoggerName": Equal("camel-k.controller.integration"),
+ "Message": Equal("Reconciling Integration"),
+ "RequestNamespace": Equal(it.Namespace),
+ "RequestName": Equal(it.Name),
+ }))
+ Expect(err).To(BeNil())
+ Expect(integrationReconciliations).To(BeNumerically(">", 0))
+
+ // Check it matches the observation in the corresponding metric
+ Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
+ PointTo(MatchFields(IgnoreExtras, Fields{
+ "Name": EqualP("camel_k_reconciliation_duration_seconds"),
+ "Help": EqualP("Camel K reconciliation loop duration"),
+ "Type": EqualP(prometheus.MetricType_HISTOGRAM),
+ "Metric": ContainElement(MatchFieldsP(IgnoreExtras, Fields{
+ "Label": ConsistOf(
+ label("group", camelv1.SchemeGroupVersion.Group),
+ label("version", camelv1.SchemeGroupVersion.Version),
+ label("kind", "Integration"),
+ label("namespace", it.Namespace),
+ label("result", "Reconciled"),
+ label("tag", ""),
+ ),
+ "Histogram": MatchFieldsP(IgnoreExtras, Fields{
+ "SampleCount": EqualP(uint64(integrationReconciliations)),
+ }),
+ })),
+ })),
+ ))
+
+ // Count the number of IntegrationKit reconciliations
+ integrationKitReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
+ "LoggerName": Equal("camel-k.controller.integrationkit"),
+ "Message": Equal("Reconciling IntegrationKit"),
+ "RequestNamespace": Equal(it.Status.IntegrationKit.Namespace),
+ "RequestName": Equal(it.Status.IntegrationKit.Name),
+ }))
+ Expect(err).To(BeNil())
+ Expect(integrationKitReconciliations).To(BeNumerically(">", 0))
+
+ // Check it matches the observation in the corresponding metric
+ Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
+ PointTo(MatchFields(IgnoreExtras, Fields{
+ "Name": EqualP("camel_k_reconciliation_duration_seconds"),
+ "Help": EqualP("Camel K reconciliation loop duration"),
+ "Type": EqualP(prometheus.MetricType_HISTOGRAM),
+ "Metric": ContainElement(MatchFieldsP(IgnoreExtras, Fields{
+ "Label": ConsistOf(
+ label("group", camelv1.SchemeGroupVersion.Group),
+ label("version", camelv1.SchemeGroupVersion.Version),
+ label("kind", "IntegrationKit"),
+ label("namespace", it.Status.IntegrationKit.Namespace),
+ label("result", "Reconciled"),
+ label("tag", ""),
+ ),
+ "Histogram": MatchFieldsP(IgnoreExtras, Fields{
+ "SampleCount": EqualP(uint64(integrationKitReconciliations)),
+ }),
+ })),
+ })),
+ ))
+
+ // Count the number of Build reconciliations
+ buildReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
+ "LoggerName": Equal("camel-k.controller.build"),
+ "Message": Equal("Reconciling Build"),
+ "RequestNamespace": Equal(build.Namespace),
+ "RequestName": Equal(build.Name),
+ }))
+ Expect(err).To(BeNil())
+
+ // Check it matches the observation in the corresponding metric
+ buildReconciled := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
+ MatchFieldsP(IgnoreExtras, Fields{
+ "Label": ConsistOf(
+ label("group", camelv1.SchemeGroupVersion.Group),
+ label("version", camelv1.SchemeGroupVersion.Version),
+ label("kind", "Build"),
+ label("namespace", build.Namespace),
+ label("result", "Reconciled"),
+ label("tag", ""),
+ ),
+ }))
+ Expect(buildReconciled).NotTo(BeNil())
+ buildReconciledCount := *buildReconciled.Histogram.SampleCount
+ Expect(buildReconciledCount).To(BeNumerically(">", 0))
+
+ buildRequeued := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
+ MatchFieldsP(IgnoreExtras, Fields{
+ "Label": ConsistOf(
+ label("group", camelv1.SchemeGroupVersion.Group),
+ label("version", camelv1.SchemeGroupVersion.Version),
+ label("kind", "Build"),
+ label("namespace", build.Namespace),
+ label("result", "Requeued"),
+ label("tag", ""),
+ ),
+ }))
+ buildRequeuedCount := uint64(0)
+ if buildRequeued != nil {
+ buildRequeuedCount = *buildRequeued.Histogram.SampleCount
+ }
+
+ Expect(buildReconciliations).To(BeNumerically("==", buildReconciledCount+buildRequeuedCount))
})
t.Run("Integration metrics", func(t *testing.T) {
@@ -177,6 +341,17 @@ func TestMetrics(t *testing.T) {
})
}
+func getMetric(family *prometheus.MetricFamily, matcher types.GomegaMatcher) *prometheus.Metric {
+ for _, metric := range family.Metric {
+ if match, err := matcher.Match(metric); err != nil {
+ panic(err)
+ } else if match {
+ return metric
+ }
+ }
+ return nil
+}
+
func label(name, value string) *prometheus.LabelPair {
return &prometheus.LabelPair{
Name: &name,
@@ -209,6 +384,14 @@ func parsePrometheusData(data []byte) (map[string]*prometheus.MetricFamily, erro
return parser.TextToMetricFamilies(bytes.NewReader(data))
}
+func EqualP(expected interface{}) types.GomegaMatcher {
+ return PointTo(Equal(expected))
+}
+
+func MatchFieldsP(options Options, fields Fields) types.GomegaMatcher {
+ return PointTo(MatchFields(options, fields))
+}
+
func stringP(s string) *string {
return &s
}
diff --git a/e2e/support/util/log_counter.go b/e2e/support/util/log_counter.go
new file mode 100644
index 0000000..02a72db
--- /dev/null
+++ b/e2e/support/util/log_counter.go
@@ -0,0 +1,47 @@
+// +build integration
+
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+ "github.com/onsi/gomega/types"
+)
+
+type LogCounter struct {
+ logs *[]LogEntry
+}
+
+func NewLogCounter(logs *[]LogEntry) *LogCounter {
+ counter := LogCounter{
+ logs: logs,
+ }
+ return &counter
+}
+
+func (w *LogCounter) Count(matcher types.GomegaMatcher) (uint, error) {
+ count := uint(0)
+ for _, log := range *w.logs {
+ if match, err := matcher.Match(log); err != nil {
+ return 0, err
+ } else if match {
+ count++
+ }
+ }
+ return count, nil
+}