You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2021/03/10 02:15:09 UTC
[servicecomb-java-chassis] branch master updated: [SCB-2187]Support
reading yaml file config from config center (#2288)
This is an automated email from the ASF dual-hosted git repository.
liubao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-java-chassis.git
The following commit(s) were added to refs/heads/master by this push:
new 3d49003 [SCB-2187]Support reading yaml file config from config center (#2288)
3d49003 is described below
commit 3d490030bb877dcb069adc2f129a263b4a910b4c
Author: wxkwxkwxk1231 <wy...@icloud.com>
AuthorDate: Wed Mar 10 10:14:57 2021 +0800
[SCB-2187]Support reading yaml file config from config center (#2288)
* read yaml from config center
* Update ProducerTestsAfterBootup.java
modify test case to fit spring-boot tests
* Update microservice.yaml
delete test case
* Update ConfigCenterConfigurationSourceImpl.java
reformat codestyle
Update ConfigCenterConfigurationSourceImpl.java
remove capture for NullPointerException
Update ConfigCenterConfigurationSourceImpl.java
* add dependency
repair null check
code style check
* correct details
correct details
* remove exception capture, make yaml2Properties more safety
* modify YAMLUtil and add Exception capture
---
.../springmvc/server/ProducerTestsAfterBootup.java | 10 ++--
.../demo/springmvc/server/ReadFileSource.java | 43 ++++++++++++++++
.../src/main/resources/microservice.yaml | 3 +-
.../ConfigCenterConfigurationSourceImpl.java | 28 ++++++++++-
.../config/client/ConfigCenterClient.java | 57 +++++++++++-----------
.../config/client/ConfigCenterConfig.java | 23 +++++++--
.../org/apache/servicecomb/config/YAMLUtil.java | 30 +++++++++++-
7 files changed, 152 insertions(+), 42 deletions(-)
diff --git a/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ProducerTestsAfterBootup.java b/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ProducerTestsAfterBootup.java
index 60ef51a..5ce24df 100644
--- a/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ProducerTestsAfterBootup.java
+++ b/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ProducerTestsAfterBootup.java
@@ -26,9 +26,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
+import com.netflix.config.DynamicPropertyFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectWriter;
-import com.netflix.config.DynamicPropertyFactory;
import io.swagger.models.Swagger;
import io.swagger.util.Yaml;
@@ -91,10 +91,10 @@ public class ProducerTestsAfterBootup implements BootListener {
public void testRegisteredBasePath() {
if (DynamicPropertyFactory.getInstance().getBooleanProperty("servicecomb.test.vert.transport", true).get()) {
- TestMgr.check(17, RegistrationManager.INSTANCE.getMicroservice().getPaths().size());
- } else {
- TestMgr.check(18, RegistrationManager.INSTANCE.getMicroservice().getPaths().size());
- }
+ TestMgr.check(18, RegistrationManager.INSTANCE.getMicroservice().getPaths().size());
+ } else {
+ TestMgr.check(19, RegistrationManager.INSTANCE.getMicroservice().getPaths().size());
+ }
}
private String getSwaggerContent(Swagger swagger) {
diff --git a/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ReadFileSource.java b/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ReadFileSource.java
new file mode 100644
index 0000000..77d7832
--- /dev/null
+++ b/demo/demo-springmvc/springmvc-server/src/main/java/org/apache/servicecomb/demo/springmvc/server/ReadFileSource.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.demo.springmvc.server;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.apache.servicecomb.provider.rest.common.RestSchema;
+
+@RestSchema(schemaId = "testFileSource")
+@RequestMapping(path = "/springmvc/fileSource")
+public class ReadFileSource {
+ @Value("${int:-1}")
+ public int testInt;
+
+ @GetMapping(path = "/int")
+ public int getTestInt() {
+ return testInt;
+ }
+
+ @Value("${String:error}")
+ public String testString;
+
+ @GetMapping(path = "/String")
+ public String getTestString() {
+ return testString;
+ }
+}
diff --git a/demo/demo-springmvc/springmvc-server/src/main/resources/microservice.yaml b/demo/demo-springmvc/springmvc-server/src/main/resources/microservice.yaml
index 1341acb..4ebec70 100644
--- a/demo/demo-springmvc/springmvc-server/src/main/resources/microservice.yaml
+++ b/demo/demo-springmvc/springmvc-server/src/main/resources/microservice.yaml
@@ -47,10 +47,11 @@ servicecomb:
# can download config center from https://cse-bucket.obs.myhwclouds.com/LocalCSE/Local-CSE-1.0.0.zip to test dynamic config
config:
client:
- # serverUri: http://127.0.0.1:30113
+ # serverUri: http://127.0.0.1:30113
refreshMode: 0
refresh_interval: 5000
refreshPort: 30114
+ fileSource: file1, file2
uploads:
directory: target
rest:
diff --git a/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/archaius/sources/ConfigCenterConfigurationSourceImpl.java b/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/archaius/sources/ConfigCenterConfigurationSourceImpl.java
index eb40703..5e047cc 100644
--- a/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/archaius/sources/ConfigCenterConfigurationSourceImpl.java
+++ b/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/archaius/sources/ConfigCenterConfigurationSourceImpl.java
@@ -19,17 +19,19 @@ package org.apache.servicecomb.config.archaius.sources;
import static com.netflix.config.WatchedUpdateResult.createIncremental;
+import java.io.ByteArrayInputStream;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.commons.configuration.Configuration;
-import org.apache.servicecomb.deployment.Deployment;
-import org.apache.servicecomb.deployment.DeploymentProvider;
import org.apache.servicecomb.config.ConfigMapping;
+import org.apache.servicecomb.config.YAMLUtil;
import org.apache.servicecomb.config.client.ConfigCenterClient;
import org.apache.servicecomb.config.client.ConfigCenterConfig;
import org.apache.servicecomb.config.spi.ConfigCenterConfigurationSource;
+import org.apache.servicecomb.deployment.Deployment;
+import org.apache.servicecomb.deployment.DeploymentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -126,7 +128,19 @@ public class ConfigCenterConfigurationSourceImpl implements ConfigCenterConfigur
if (parseConfigs == null || parseConfigs.isEmpty()) {
return;
}
+
Map<String, Object> configuration = ConfigMapping.getConvertedMap(parseConfigs);
+ if (configCenterClient != null) {
+ List<String> fileSourceList = configCenterClient.getFileSources();
+ if (fileSourceList != null) {
+ fileSourceList.forEach(fileName -> {
+ if (configuration.containsKey(fileName)) {
+ replaceConfig(configuration, fileName);
+ }
+ });
+ }
+ }
+
if ("create".equals(action)) {
valueCache.putAll(configuration);
updateConfiguration(createIncremental(ImmutableMap.<String, Object>copyOf(configuration),
@@ -147,5 +161,15 @@ public class ConfigCenterConfigurationSourceImpl implements ConfigCenterConfigur
}
LOGGER.info("Config value cache changed: action:{}; item:{}", action, configuration.keySet());
}
+
+ private void replaceConfig(Map<String, Object> configuration, String fileName) {
+ Object tempConfig = configuration.get(fileName);
+ try {
+ Map<String, Object> properties = YAMLUtil.yaml2Properties(tempConfig.toString());
+ configuration.putAll(properties);
+ } catch (Exception e) {
+ LOGGER.warn("yaml file has incorrect format", e);
+ }
+ }
}
}
diff --git a/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterClient.java b/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterClient.java
index 26e7314..290a6d4 100644
--- a/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterClient.java
+++ b/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterClient.java
@@ -17,25 +17,15 @@
package org.apache.servicecomb.config.client;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.vertx.core.MultiMap;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpClientRequest;
+import io.vertx.core.http.WebSocket;
+import io.vertx.core.http.WebSocketConnectOptions;
+import io.vertx.core.http.impl.FrameType;
+import io.vertx.core.http.impl.ws.WebSocketFrameImpl;
import org.apache.commons.lang.StringUtils;
import org.apache.servicecomb.config.archaius.sources.ConfigCenterConfigurationSourceImpl;
import org.apache.servicecomb.foundation.auth.AuthHeaderLoader;
@@ -49,16 +39,19 @@ import org.apache.servicecomb.foundation.vertx.client.http.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.core.type.TypeReference;
-
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.vertx.core.MultiMap;
-import io.vertx.core.buffer.Buffer;
-import io.vertx.core.http.HttpClientRequest;
-import io.vertx.core.http.WebSocket;
-import io.vertx.core.http.WebSocketConnectOptions;
-import io.vertx.core.http.impl.FrameType;
-import io.vertx.core.http.impl.ws.WebSocketFrameImpl;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
public class ConfigCenterClient {
@@ -87,6 +80,12 @@ public class ConfigCenterClient {
private String serviceName = CONFIG_CENTER_CONFIG.getServiceName();
private String environment = CONFIG_CENTER_CONFIG.getEnvironment();
+
+ private List<String> fileSources = CONFIG_CENTER_CONFIG.getFileSources();
+
+ public List<String> getFileSources() {
+ return fileSources;
+ }
private MemberDiscovery memberDiscovery = new MemberDiscovery(CONFIG_CENTER_CONFIG.getServerUri());
diff --git a/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterConfig.java b/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterConfig.java
index 609dae9..6ff5747 100644
--- a/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterConfig.java
+++ b/dynamic-config/config-cc/src/main/java/org/apache/servicecomb/config/client/ConfigCenterConfig.java
@@ -17,15 +17,15 @@
package org.apache.servicecomb.config.client;
-import java.util.List;
-
+import com.google.common.base.Joiner;
+import com.netflix.config.ConcurrentCompositeConfiguration;
import org.apache.servicecomb.config.BootStrapProperties;
import org.apache.servicecomb.deployment.Deployment;
import org.apache.servicecomb.deployment.DeploymentProvider;
import org.apache.servicecomb.foundation.vertx.VertxConst;
-import com.google.common.base.Joiner;
-import com.netflix.config.ConcurrentCompositeConfiguration;
+import java.util.ArrayList;
+import java.util.List;
public final class ConfigCenterConfig {
public static final ConfigCenterConfig INSTANCE = new ConfigCenterConfig();
@@ -53,6 +53,8 @@ public final class ConfigCenterConfig {
public static final String CONNECTION_TIME_OUT = "servicecomb.config.client.timeout.connection";
public static final String EVENT_LOOP_SIZE = "servicecomb.config.client.eventLoopSize";
+
+ public static final String FILE_SOURCE = "servicecomb.config.client.fileSource";
public static final String VERTICAL_INSTANCE_COUNT = "servicecomb.config.client.verticalInstanceCount";
@@ -100,6 +102,19 @@ public final class ConfigCenterConfig {
public String getApiVersion() {
return finalConfig.getString(URI_API_VERSION, "v3");
}
+
+ @SuppressWarnings("unchecked")
+ public List<String> getFileSources() {
+ Object property = finalConfig.getProperty(FILE_SOURCE);
+ if (property instanceof String) {
+ List<String> result = new ArrayList<>();
+ result.add((String) property);
+ return result;
+ } else if (property instanceof List) {
+ return (List<String>) property;
+ }
+ return new ArrayList<>();
+ }
public int getRefreshInterval() {
return finalConfig.getInt(REFRESH_INTERVAL, DEFAULT_REFRESH_INTERVAL);
diff --git a/foundations/foundation-config/src/main/java/org/apache/servicecomb/config/YAMLUtil.java b/foundations/foundation-config/src/main/java/org/apache/servicecomb/config/YAMLUtil.java
index e37b4e2..fc0de9c 100644
--- a/foundations/foundation-config/src/main/java/org/apache/servicecomb/config/YAMLUtil.java
+++ b/foundations/foundation-config/src/main/java/org/apache/servicecomb/config/YAMLUtil.java
@@ -24,6 +24,8 @@ import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.Constructor;
@@ -35,6 +37,8 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
public final class YAMLUtil {
private static final Yaml SAFE_PARSER = new Yaml(new SafeConstructor());
+ private static final Logger LOGGER = LoggerFactory.getLogger(YAMLUtil.class);
+
private YAMLUtil() {
}
@@ -50,7 +54,31 @@ public final class YAMLUtil {
@SuppressWarnings("unchecked")
public static Map<String, Object> yaml2Properties(InputStream input) {
Map<String, Object> configurations = new LinkedHashMap<>();
- SAFE_PARSER.loadAll(input).forEach(data -> configurations.putAll(retrieveItems("", (Map<String, Object>) data)));
+ SAFE_PARSER.loadAll(input).forEach(data -> {
+ if (data instanceof Map) {
+ configurations.putAll(retrieveItems("", (Map<String, Object>) data));
+ } else {
+ LOGGER.warn("input cannot be convert to map");
+ }
+ });
+ return configurations;
+ }
+
+ /**
+ * load a input {@link String} to be a map {@link Map}
+ * @param input the String to be loaded
+ * @return a config map
+ */
+ @SuppressWarnings("unchecked")
+ public static Map<String, Object> yaml2Properties(String input) {
+ Map<String, Object> configurations = new LinkedHashMap<>();
+ SAFE_PARSER.loadAll(input).forEach(data -> {
+ if (data instanceof Map) {
+ configurations.putAll(retrieveItems("", (Map<String, Object>) data));
+ } else {
+ LOGGER.warn("input cannot be convert to map");
+ }
+ });
return configurations;
}