You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by as...@apache.org on 2021/03/12 15:39:33 UTC
[camel-k] 02/10: test: Introduce log 'sky' walker
This is an automated email from the ASF dual-hosted git repository.
astefanutti pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit ddddb63c2db71f6c6e90484a9c9b2753bf5a07f1
Author: Antonin Stefanutti <an...@stefanutti.fr>
AuthorDate: Tue Feb 23 19:07:03 2021 +0100
test: Introduce log 'sky' walker
---
e2e/common/metrics_test.go | 52 ++++++++++++++++------------
e2e/support/test_support.go | 43 ++----------------------
e2e/support/util/log_walker.go | 69 ++++++++++++++++++++++++++++++++++++++
e2e/support/util/structured_log.go | 61 +++++++++++++++++++++++++++++++++
4 files changed, 163 insertions(+), 62 deletions(-)
diff --git a/e2e/common/metrics_test.go b/e2e/common/metrics_test.go
index c46b72e..57864be 100644
--- a/e2e/common/metrics_test.go
+++ b/e2e/common/metrics_test.go
@@ -25,7 +25,6 @@ import (
"bytes"
"fmt"
"math"
- "strings"
"testing"
"time"
@@ -39,6 +38,7 @@ import (
"github.com/prometheus/common/expfmt"
. "github.com/apache/camel-k/e2e/support"
+ . "github.com/apache/camel-k/e2e/support/util"
camelv1 "github.com/apache/camel-k/pkg/apis/camel/v1"
)
@@ -76,7 +76,35 @@ func TestMetrics(t *testing.T) {
Expect(err).To(BeNil())
// Check it's consistent with the duration observed from logs
- durationFromLogs := buildDuration(&logs, ns, build.Name)
+ var ts1, ts2 time.Time
+ err = NewLogWalker(&logs).
+ AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ "LoggerName": Equal("camel-k.controller.build"),
+ "Message": Equal("Build state transition"),
+ "Phase": Equal("Pending"),
+ "RequestName": Equal(build.Name),
+ }), LogEntryNoop).
+ AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ "LoggerName": Equal("camel-k.controller.build"),
+ "Message": Equal("Reconciling Build"),
+ "RequestName": Equal(build.Name),
+ }), func(l *LogEntry) { ts1 = l.Timestamp.Time }).
+ AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ "LoggerName": Equal("camel-k.builder"),
+ "Message": HavePrefix("resolved image"),
+ }), LogEntryNoop).
+ AddStep(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+ "LoggerName": Equal("camel-k.controller.build"),
+ "Message": Equal("Reconciling Build"),
+ "RequestName": Equal(build.Name),
+ }), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
+ Walk()
+ Expect(err).To(BeNil())
+ Expect(ts1).NotTo(BeNil())
+ Expect(ts2).NotTo(BeNil())
+ Expect(ts2).To(BeTemporally(">", ts1))
+
+ durationFromLogs := ts2.Sub(ts1)
Expect(math.Abs((durationFromLogs - duration).Seconds())).To(BeNumerically("<", 1))
// Check the duration is observed in the corresponding metric
@@ -120,26 +148,6 @@ func TestMetrics(t *testing.T) {
})
}
-func buildDuration(logs *[]LogEntry, ns, buildName string) time.Duration {
- var ts1, ts2 time.Time
- for _, log := range *logs {
- if ts1.IsZero() &&
- log.LoggerName == "camel-k.controller.build" &&
- log.Message == "Build state transition" &&
- log.RequestNamespace == ns &&
- log.RequestName == buildName &&
- log.Phase == "Pending" {
- ts1 = log.Timestamp.Time
- }
- if ts2.IsZero() &&
- log.LoggerName == "camel-k.builder" &&
- strings.HasPrefix(log.Message, "resolved image") {
- ts2 = log.Timestamp.Time
- }
- }
- return ts2.Sub(ts1)
-}
-
func label(name, value string) *prometheus.LabelPair {
return &prometheus.LabelPair{
Name: &name,
diff --git a/e2e/support/test_support.go b/e2e/support/test_support.go
index c44bd54..4431d0b 100644
--- a/e2e/support/test_support.go
+++ b/e2e/support/test_support.go
@@ -29,10 +29,8 @@ import (
"fmt"
"io"
"io/ioutil"
- "math"
"os"
"os/exec"
- "strconv"
"strings"
"testing"
"time"
@@ -41,8 +39,6 @@ import (
"github.com/onsi/gomega"
"github.com/spf13/cobra"
- "go.uber.org/zap/zapcore"
-
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/batch/v1beta1"
coordination "k8s.io/api/coordination/v1"
@@ -253,40 +249,7 @@ func Logs(ns, podName string, options corev1.PodLogOptions) func() string {
}
}
-type Time struct {
- time.Time
-}
-
-func (t *Time) UnmarshalJSON(s []byte) (err error) {
- f, err := strconv.ParseFloat(string(s), 10)
- if err != nil {
- return err
- }
- ns := (f - math.Floor(f)) * 1000000000
- *t = Time{
- time.Unix(int64(f), int64(ns)),
- }
- return nil
-}
-
-type LogEntry struct {
- // Zap
- Level zapcore.Level `json:"level,omitempty"`
- Timestamp Time `json:"ts,omitempty"`
- LoggerName string `json:"logger,omitempty"`
- Message string `json:"msg,omitempty"`
- // Controller runtime
- RequestNamespace string `json:"request-namespace,omitempty"`
- RequestName string `json:"request-name,omitempty"`
- ApiVersion string `json:"api-version,omitempty"`
- Kind string `json:"kind,omitempty"`
- // Camel K
- Namespace string `json:"ns,omitempty"`
- Name string `json:"name,omitempty"`
- Phase string `json:"phase,omitempty"`
-}
-
-func StructuredLogs(ns, podName string, options corev1.PodLogOptions) []LogEntry {
+func StructuredLogs(ns, podName string, options corev1.PodLogOptions) []util.LogEntry {
byteReader, err := TestClient().CoreV1().Pods(ns).GetLogs(podName, &options).Stream(TestContext)
if err != nil {
log.Error(err, "Error while reading container logs")
@@ -298,10 +261,10 @@ func StructuredLogs(ns, podName string, options corev1.PodLogOptions) []LogEntry
}
}()
- entries := make([]LogEntry, 0)
+ entries := make([]util.LogEntry, 0)
scanner := bufio.NewScanner(byteReader)
for scanner.Scan() {
- entry := LogEntry{}
+ entry := util.LogEntry{}
t := scanner.Text()
err := json.Unmarshal([]byte(t), &entry)
if err != nil {
diff --git a/e2e/support/util/log_walker.go b/e2e/support/util/log_walker.go
new file mode 100644
index 0000000..3317ef7
--- /dev/null
+++ b/e2e/support/util/log_walker.go
@@ -0,0 +1,69 @@
+// +build integration
+
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+ "github.com/onsi/gomega/types"
+)
+
+var LogEntryNoop = func(*LogEntry) {}
+
+type LogWalker struct {
+ logs *[]LogEntry
+ steps []logWalkerStep
+}
+
+type logWalkerStep struct {
+ matcher types.GomegaMatcher
+ consumer func(*LogEntry)
+}
+
+func NewLogWalker(logs *[]LogEntry) *LogWalker {
+ walker := LogWalker{
+ logs: logs,
+ }
+ return &walker
+}
+
+func (w *LogWalker) AddStep(m types.GomegaMatcher, c func(*LogEntry)) *LogWalker {
+ w.steps = append(w.steps, logWalkerStep{
+ matcher: m,
+ consumer: c,
+ })
+ return w
+}
+
+func (w *LogWalker) Walk() error {
+ i := 0
+ step := w.steps[i]
+ for _, log := range *w.logs {
+ if match, err := step.matcher.Match(log); err != nil {
+ return err
+ } else if match {
+ step.consumer(&log)
+ if i == len(w.steps)-1 {
+ break
+ }
+ i++
+ step = w.steps[i]
+ }
+ }
+ return nil
+}
diff --git a/e2e/support/util/structured_log.go b/e2e/support/util/structured_log.go
new file mode 100644
index 0000000..33972aa
--- /dev/null
+++ b/e2e/support/util/structured_log.go
@@ -0,0 +1,61 @@
+// +build integration
+
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+ "math"
+ "strconv"
+ "time"
+
+ "go.uber.org/zap/zapcore"
+)
+
+type Time struct {
+ time.Time
+}
+
+func (t *Time) UnmarshalJSON(s []byte) (err error) {
+ f, err := strconv.ParseFloat(string(s), 10)
+ if err != nil {
+ return err
+ }
+ ns := (f - math.Floor(f)) * 1000000000
+ *t = Time{
+ time.Unix(int64(f), int64(ns)),
+ }
+ return nil
+}
+
+type LogEntry struct {
+ // Zap
+ Level zapcore.Level `json:"level,omitempty"`
+ Timestamp Time `json:"ts,omitempty"`
+ LoggerName string `json:"logger,omitempty"`
+ Message string `json:"msg,omitempty"`
+ // Controller runtime
+ RequestNamespace string `json:"request-namespace,omitempty"`
+ RequestName string `json:"request-name,omitempty"`
+ ApiVersion string `json:"api-version,omitempty"`
+ Kind string `json:"kind,omitempty"`
+ // Camel K
+ Namespace string `json:"ns,omitempty"`
+ Name string `json:"name,omitempty"`
+ Phase string `json:"phase,omitempty"`
+}