You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/01/27 07:53:36 UTC

[skywalking-satellite] branch main updated: override config by env (#20)

This is an automated email from the ASF dual-hosted git repository.

wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-satellite.git


The following commit(s) were added to refs/heads/main by this push:
     new 456e87a  override config by env (#20)
456e87a is described below

commit 456e87aa3014a4121ff0bdcc83877432b29dba8f
Author: Evan <31...@users.noreply.github.com>
AuthorDate: Wed Jan 27 15:53:25 2021 +0800

    override config by env (#20)
---
 configs/satellite_config.yaml                     |  34 ++--
 internal/satellite/config/loader.go               |   6 +-
 internal/satellite/config/override_by_env.go      | 114 +++++++++++++
 internal/satellite/config/override_by_env_test.go | 194 ++++++++++++++++++++++
 internal/satellite/config/satellite_config.go     |   4 +-
 5 files changed, 332 insertions(+), 20 deletions(-)

diff --git a/configs/satellite_config.yaml b/configs/satellite_config.yaml
index fc48810..323e945 100644
--- a/configs/satellite_config.yaml
+++ b/configs/satellite_config.yaml
@@ -15,27 +15,31 @@
 # limitations under the License.
 #
 
+# The logger configuration.
 logger:
-  log_pattern: "%time [%level][%field] - %msg"
-  time_pattern: "2006-01-02 15:04:05.000"
-  level: "info"
+  log_pattern: ${SATELLITE_LOGGER_LOG_PATTERN:%time [%level][%field] - %msg}
+  time_pattern: ${SATELLITE_LOGGER_TIME_PATTERN:2006-01-02 15:04:05.000}
+  level: ${SATELLITE_LOGGER_LEVEL:info}
 
+# The Satellite self telemetry configuration.
 telemetry:
-  cluster: cluster1
-  service: service1
-  instance: instance1
+  cluster: ${SATELLITE_TELEMETRY_CLUSTER:default-cluster}
+  service: ${SATELLITE_TELEMETRY_SERVICE:default-service}
+  instance: ${SATELLITE_TELEMETRY_SERVICE:default-instance}
 
+# The sharing plugins referenced by the specific plugins in the different pipes.
 sharing:
   common_config:
     pipe_name: sharing
   clients:
     - plugin_name: "kafka-client"
-      brokers: 127.0.0.1:9092
-      version: 2.1.1
+      brokers: ${SATELLITE_KAFKA_CLIENT_BROKERS:127.0.0.1:9092}
+      version: ${SATELLITE_KAFKA_VERSION:"2.1.1"}
   servers:
     - plugin_name: "grpc-server"
     - plugin_name: "prometheus-server"
-      address: ":8090"
+      address: ${SATELLITE_PROMETHEUS_ADDRESS:":8090"}
+# The working pipes.
 pipes:
   - common_config:
       pipe_name: pipe1
@@ -45,18 +49,18 @@ pipes:
         plugin_name: "grpc-nativelog-receiver"
       queue:
         plugin_name: "mmap-queue"
-        segment_size: 524288
-        max_in_mem_segments: 6
+        segment_size: ${SATELLITE_MMAP_QUEUE_SIZE:524288}
+        max_in_mem_segments: ${SATELLITE_MMAP_QUEUE_MAX_IN_MEM_SEGMENTS:6}
         queue_dir: "pipe1-log-grpc-receiver-queue"
     processor:
       filters:
     sender:
       fallbacker:
         plugin_name: none-fallbacker
-      flush_time: 1000
-      max_buffer_size: 200
-      min_flush_events: 100
+      flush_time: ${SATELLITE_PIPE1_SENDER_FLUSH_TIME:1000}
+      max_buffer_size: ${SATELLITE_PIPE1_SENDER_MAX_BUFFER_SIZE:200}
+      min_flush_events: ${SATELLITE_PIPE1_SENDER_MIN_FLUSH_EVENTS:100}
       client_name: kafka-client
       forwarders:
         - plugin_name: nativelog-kafka-forwarder
-          topic: log-topic
\ No newline at end of file
+          topic: ${SATELLITE_NATIVELOG-TOPIC:log-topic}
diff --git a/internal/satellite/config/loader.go b/internal/satellite/config/loader.go
index 21c92f8..11c6032 100644
--- a/internal/satellite/config/loader.go
+++ b/internal/satellite/config/loader.go
@@ -61,14 +61,14 @@ func load(configPath string) (*SatelliteConfig, error) {
 		return nil, err
 	}
 	v := viper.New()
-	v.AutomaticEnv()
-	v.SetEnvPrefix("satellite")
-	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	v.SetConfigType("yaml")
 	cfg := SatelliteConfig{}
 	if err := v.ReadConfig(bytes.NewReader(content)); err != nil {
 		return nil, err
 	}
+	if err := overrideConfigByEnv(v); err != nil {
+		return nil, fmt.Errorf("cannot override value by env config: %v", err)
+	}
 	if err := v.Unmarshal(&cfg); err != nil {
 		return nil, err
 	}
diff --git a/internal/satellite/config/override_by_env.go b/internal/satellite/config/override_by_env.go
new file mode 100644
index 0000000..6ae678f
--- /dev/null
+++ b/internal/satellite/config/override_by_env.go
@@ -0,0 +1,114 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package config
+
+import (
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/viper"
+)
+
+// ${ENV:defaultVal} pattern is supported to override the default value by the env config.
+// And the value would be converted to the different types, such as int, float64 ana string.
+// If you want to keep it as the string value, please decorate it with `"`.
+const RegularExpression = "${_|[A-Z]+:.*}"
+
+func overrideConfigByEnv(v *viper.Viper) error {
+	regex, err := regexp.Compile(RegularExpression)
+	if err != nil {
+		return err
+	}
+	overrideCfg := overrideMapStringInterface(v.AllSettings(), regex)
+	for key, val := range overrideCfg {
+		v.Set(key, val)
+	}
+	return nil
+}
+
+func overrideMapStringInterface(cfg map[string]interface{}, regex *regexp.Regexp) map[string]interface{} {
+	res := make(map[string]interface{})
+	for key, val := range cfg {
+		switch val := val.(type) {
+		case string:
+			res[key] = overrideString(val, regex)
+		case []interface{}:
+			res[key] = overrideSlice(val, regex)
+		case map[string]interface{}:
+			res[key] = overrideMapStringInterface(val, regex)
+		case map[interface{}]interface{}:
+			res[key] = overrideMapInterfaceInterface(val, regex)
+		default:
+			res[key] = val
+		}
+	}
+	return res
+}
+
+func overrideSlice(m []interface{}, regex *regexp.Regexp) []interface{} {
+	res := make([]interface{}, 0)
+	for _, val := range m {
+		switch val := val.(type) {
+		case map[string]interface{}:
+			res = append(res, overrideMapStringInterface(val, regex))
+		case map[interface{}]interface{}:
+			res = append(res, overrideMapInterfaceInterface(val, regex))
+		}
+	}
+	return res
+}
+
+func overrideMapInterfaceInterface(m map[interface{}]interface{}, regex *regexp.Regexp) interface{} {
+	cfg := make(map[string]interface{})
+	for key, val := range m {
+		cfg[key.(string)] = val
+	}
+	return overrideMapStringInterface(cfg, regex)
+}
+
+func overrideString(s string, regex *regexp.Regexp) interface{} {
+	if regex.MatchString(s) {
+		index := strings.Index(s, ":")
+		envName := s[2:index]
+		defaultVal := s[index+1 : len(s)-1]
+		envVal := os.Getenv(envName)
+		if envVal != "" {
+			defaultVal = envVal
+		}
+		return parseVal(defaultVal)
+	}
+	return s
+}
+
+func parseVal(val string) interface{} {
+	if intVal, err := strconv.Atoi(val); err == nil {
+		return intVal
+	} else if floatVal, err := strconv.ParseFloat(val, 64); err == nil {
+		return floatVal
+	} else if strings.EqualFold(val, "true") {
+		return true
+	} else if strings.EqualFold(val, "false") {
+		return false
+	} else if strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"") {
+		return val[1 : len(val)-1]
+	} else {
+		return val
+	}
+}
diff --git a/internal/satellite/config/override_by_env_test.go b/internal/satellite/config/override_by_env_test.go
new file mode 100644
index 0000000..972579c
--- /dev/null
+++ b/internal/satellite/config/override_by_env_test.go
@@ -0,0 +1,194 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package config
+
+import (
+	"os"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func Test_overrideString(t *testing.T) {
+	type args struct {
+		expression string
+		env        string
+	}
+	tests := []struct {
+		name string
+		args args
+		want interface{}
+	}{
+		{
+			name: "override_string",
+			args: args{
+				expression: "${TEST_OVERRIDE_STRING:test_str}",
+				env:        "TEST_OVERRIDE_STRING=test_override_string",
+			},
+			want: "test_override_string",
+		},
+		{
+			name: "no_override_string",
+			args: args{
+				expression: "${TEST_NO_OVERRIDE_STRING:test_str}",
+			},
+			want: "test_str",
+		},
+		{
+			name: "no_override_false",
+			args: args{
+				expression: "${TEST_NO_OVERRIDE_FALSE:false}",
+			},
+			want: false,
+		},
+		{
+			name: "no_override_true",
+			args: args{
+				expression: "${TEST_NO_OVERRIDE_TRUE:true}",
+			},
+			want: true,
+		},
+		{
+			name: "override_boolean",
+			args: args{
+				expression: "${TEST_OVERRIDE_BOOLEAN:true}",
+				env:        "TEST_OVERRIDE_BOOLEAN=false",
+			},
+			want: false,
+		},
+		{
+			name: "no_override_int",
+			args: args{
+				expression: "${TEST_OVERRIDE_INT:10}",
+			},
+			want: 10,
+		},
+		{
+			name: "override_int",
+			args: args{
+				expression: "${TEST_OVERRIDE_INT:10}",
+				env:        "TEST_OVERRIDE_INT=15",
+			},
+			want: 15,
+		},
+		{
+			name: "override_float",
+			args: args{
+				expression: "${TEST_OVERRIDE_FLOAT:10.5}",
+				env:        "TEST_OVERRIDE_FLOAT=15.7",
+			},
+			want: 15.7,
+		},
+		{
+			name: "no_override_force_string",
+			args: args{
+				expression: "${TEST_NO_OVERRIDE_FORCE_STRING:\"10.5\"}",
+			},
+			want: "10.5",
+		},
+	}
+	for _, tt := range tests {
+		if tt.args.env != "" {
+			envArr := strings.Split(tt.args.env, "=")
+			if err := os.Setenv(envArr[0], envArr[1]); err != nil {
+				t.Fatalf("cannot set the env %s  config: %v", tt.args.env, err)
+			}
+		}
+		regex, err := regexp.Compile(RegularExpression)
+		if err != nil {
+			t.Fatalf("cannot generate the regular expression: %s", RegularExpression)
+		}
+		t.Run(tt.name, func(t *testing.T) {
+			if got := overrideString(tt.args.expression, regex); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("overrideString() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_overrideMapStringInterface(t *testing.T) {
+	type args struct {
+		cfg map[string]interface{}
+		env map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[string]interface{}
+	}{
+		{
+			name: "test-overrideByEnv",
+			args: args{
+				cfg: map[string]interface{}{
+					"stringKey": "${OVERRIDE_STRING_KEY:stringKey}",
+					"intKey":    "${OVERRIDE_INT_KEY:10}",
+					"boolKey":   "${OVERRIDE_BOOL_KEY:false}",
+					"mapKey": map[string]interface{}{
+						"mapStringKey": "${OVERRIDE_STRING_KEY:stringKey}",
+						"mapIntKey":    "${OVERRIDE_INT_KEY:10}",
+						"mapBoolKey":   "${OVERRIDE_BOOL_KEY:false}",
+						"mapInterfaceKey": map[interface{}]interface{}{
+							"mapinterfaceStringKey": "${OVERRIDE_STRING_KEY:stringKey}",
+							"mapinterfaceIntKey":    "${OVERRIDE_INT_KEY:10}",
+							"mapinterfaceBoolKey":   "${OVERRIDE_BOOL_KEY:false}",
+						},
+					},
+				},
+				env: map[string]string{
+					"OVERRIDE_STRING_KEY": "env-string",
+					"OVERRIDE_INT_KEY":    "100",
+					"OVERRIDE_BOOL_KEY":   "true",
+				},
+			},
+			want: map[string]interface{}{
+				"stringKey": "env-string",
+				"intKey":    100,
+				"boolKey":   true,
+				"mapKey": map[string]interface{}{
+					"mapStringKey": "env-string",
+					"mapIntKey":    100,
+					"mapBoolKey":   true,
+					"mapInterfaceKey": map[string]interface{}{
+						"mapinterfaceStringKey": "env-string",
+						"mapinterfaceIntKey":    100,
+						"mapinterfaceBoolKey":   true,
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		for k, v := range tt.args.env {
+			err := os.Setenv(k, v)
+			if err != nil {
+				t.Fatalf("cannot set the env config{%s=%s}: %v", k, v, err)
+			}
+		}
+		t.Run(tt.name, func(t *testing.T) {
+			regex, _ := regexp.Compile(RegularExpression)
+			got := overrideMapStringInterface(tt.args.cfg, regex)
+			if !cmp.Equal(got, tt.want) {
+				t.Errorf("overrideConfigByEnv()  got = %v, want = %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/internal/satellite/config/satellite_config.go b/internal/satellite/config/satellite_config.go
index 3033692..2828285 100644
--- a/internal/satellite/config/satellite_config.go
+++ b/internal/satellite/config/satellite_config.go
@@ -37,15 +37,15 @@ type SatelliteConfig struct {
 
 // SharingConfig contains some plugins,which could be shared by every namespace. That is useful to reduce resources cost.
 type SharingConfig struct {
-	SharingCommonConfig config.CommonFields `mapstructure:"common_config"`
 	Clients             []plugin.Config     `mapstructure:"clients"`
 	Servers             []plugin.Config     `mapstructure:"servers"`
+	SharingCommonConfig config.CommonFields `mapstructure:"common_config"`
 }
 
 // PipeConfig initializes the different module in different namespace.
 type PipeConfig struct {
-	PipeCommonConfig config.CommonFields        `mapstructure:"common_config"`
 	Gatherer         *gatherer.GathererConfig   `mapstructure:"gatherer"`
+	PipeCommonConfig config.CommonFields        `mapstructure:"common_config"`
 	Processor        *processor.ProcessorConfig `mapstructure:"processor"`
 	Sender           *sender.SenderConfig       `mapstructure:"sender"`
 }