You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2019/03/26 04:02:42 UTC
[incubator-skywalking] branch master updated: Support endpoint
group in mesh. (#2403)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 3acd044 Support endpoint group in mesh. (#2403)
3acd044 is described below
commit 3acd044d25c9c9adbc2c3073dd34ac2602711c8b
Author: 吴晟 Wu Sheng <wu...@foxmail.com>
AuthorDate: Mon Mar 25 21:02:35 2019 -0700
Support endpoint group in mesh. (#2403)
* Support endpoint group in mesh.
* Fix CI.
* Refactor for reduce string compare.
* Support endpoint naming rule in service level
---
.../skywalking/apm/util/StringFormatGroup.java | 101 +++++++++++++++
.../skywalking/apm/util/StringFormatGroupTest.java | 136 +++++++++++++++++++++
.../server/receiver/mesh/EndpointNameFormater.java | 67 ++++++++++
.../receiver/mesh/TelemetryDataDispatcher.java | 10 +-
.../resources/endpoint_naming_rules.properties | 19 +++
pom.xml | 14 +++
6 files changed, 346 insertions(+), 1 deletion(-)
diff --git a/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java b/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java
new file mode 100644
index 0000000..36d689b
--- /dev/null
+++ b/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.util;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+/**
+ * Group patterns use {@link java.util.regex.Pattern} as core, could group the input strings to matched group or return
+ * original string.
+ *
+ * @author wusheng
+ */
+public class StringFormatGroup {
+ private final List<PatternRule> rules;
+
+ public StringFormatGroup() {
+ rules = new ArrayList<PatternRule>();
+ }
+
+ /**
+ * Add a new match rule. The rule will follow the order of being added.
+ *
+ * @param name will be used when ruleRegex matched.
+ * @param ruleRegex to match target string.
+ */
+ public void addRule(String name, String ruleRegex) {
+ if (rules.contains(name)) {
+ return;
+ }
+ PatternRule rule = new PatternRule(name, ruleRegex);
+ rules.add(rule);
+ }
+
+ /**
+ * Format the string based on rules.
+ *
+ * @param string to be formatted
+ * @return matched rule name, or original string.
+ */
+ public FormatResult format(String string) {
+ for (PatternRule rule : rules) {
+ if (rule.getPattern().matcher(string).matches()) {
+ return new FormatResult(true, rule.getName());
+ }
+ }
+ return new FormatResult(false, string);
+ }
+
+ public class FormatResult {
+ private boolean match;
+ private String name;
+
+ public FormatResult(boolean match, String name) {
+ this.match = match;
+ this.name = name;
+ }
+
+ public boolean isMatch() {
+ return match;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ private class PatternRule {
+ private String name;
+ private Pattern pattern;
+
+ private PatternRule(String name, String ruleRegex) {
+ this.name = name;
+ pattern = Pattern.compile(ruleRegex);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Pattern getPattern() {
+ return pattern;
+ }
+ }
+}
diff --git a/apm-commons/apm-util/src/test/java/org/apache/skywalking/apm/util/StringFormatGroupTest.java b/apm-commons/apm-util/src/test/java/org/apache/skywalking/apm/util/StringFormatGroupTest.java
new file mode 100644
index 0000000..4640220
--- /dev/null
+++ b/apm-commons/apm-util/src/test/java/org/apache/skywalking/apm/util/StringFormatGroupTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.util;
+
+import java.util.concurrent.TimeUnit;
+import org.junit.*;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.runner.*;
+import org.openjdk.jmh.runner.options.*;
+
+public class StringFormatGroupTest {
+ @Benchmark
+ @Test
+ public void testMatch() {
+ StringFormatGroup group = new StringFormatGroup();
+ group.addRule("/name/*/add", "/name/.+/add");
+ Assert.assertEquals("/name/*/add", group.format("/name/test/add").getName());
+
+ group = new StringFormatGroup();
+ group.addRule("/name/*/add/{orderId}", "/name/.+/add/.*");
+ Assert.assertEquals("/name/*/add/{orderId}", group.format("/name/test/add/12323").getName());
+ }
+
+ @Benchmark
+ @Test
+ public void test100Rule() {
+ StringFormatGroup group = new StringFormatGroup();
+ group.addRule("/name/*/add/{orderId}", "/name/.+/add/.*");
+ for (int i = 0; i < 100; i++) {
+ group.addRule("/name/*/add/{orderId}" + "/" + 1, "/name/.+/add/.*" + "/abc");
+ }
+ Assert.assertEquals("/name/*/add/{orderId}", group.format("/name/test/add/12323").getName());
+ }
+
+ /**
+ * The report below shows this pattern match performance is much about rule numbers.
+ * This is a single thread test.
+ *
+ * @throws RunnerException
+ */
+ @BenchmarkMode(Mode.AverageTime)
+ @OutputTimeUnit(TimeUnit.MICROSECONDS)
+ public void performanceBenchmark() throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(StringFormatGroupTest.class.getSimpleName())
+ .forks(1)
+ .warmupIterations(0)
+ .measurementIterations(5)
+ .build();
+
+ new Runner(opt).run();
+ }
+
+ /*********************************
+ * # JMH version: 1.21
+ * # VM version: JDK 1.8.0_91, Java HotSpot(TM) 64-Bit Server VM, 25.91-b14
+ * # VM invoker: /Users/wusheng/Documents/applications/jdk1.8.0_91.jdk/Contents/Home/jre/bin/java
+ * # VM options: -ea -Didea.test.cyclic.buffer.size=1048576 -javaagent:/Applications/IntelliJ IDEA.app/Contents/lib/idea_rt.jar=54841:/Applications/IntelliJ IDEA.app/Contents/bin -Dfile.encoding=UTF-8
+ * # Warmup: <none>
+ * # Measurement: 5 iterations, 10 s each
+ * # Timeout: 10 min per iteration
+ * # Threads: 1 thread, will synchronize iterations
+ * # Benchmark mode: Throughput, ops/time
+ * # Benchmark: org.apache.skywalking.apm.util.StringFormatGroupTest.test100Rule
+ *
+ * # Run progress: 0.00% complete, ETA 00:01:40
+ * # Fork: 1 of 1
+ * Iteration 1: 32016.496 ops/s
+ * Iteration 2: 36703.873 ops/s
+ * Iteration 3: 37121.543 ops/s
+ * Iteration 4: 36898.225 ops/s
+ * Iteration 5: 34712.564 ops/s
+ *
+ *
+ * Result "org.apache.skywalking.apm.util.StringFormatGroupTest.test100Rule":
+ * 35490.540 ±(99.9%) 8345.368 ops/s [Average]
+ * (min, avg, max) = (32016.496, 35490.540, 37121.543), stdev = 2167.265
+ * CI (99.9%): [27145.173, 43835.908] (assumes normal distribution)
+ *
+ *
+ * # JMH version: 1.21
+ * # VM version: JDK 1.8.0_91, Java HotSpot(TM) 64-Bit Server VM, 25.91-b14
+ * # VM invoker: /Users/wusheng/Documents/applications/jdk1.8.0_91.jdk/Contents/Home/jre/bin/java
+ * # VM options: -ea -Didea.test.cyclic.buffer.size=1048576 -javaagent:/Applications/IntelliJ IDEA.app/Contents/lib/idea_rt.jar=54841:/Applications/IntelliJ IDEA.app/Contents/bin -Dfile.encoding=UTF-8
+ * # Warmup: <none>
+ * # Measurement: 5 iterations, 10 s each
+ * # Timeout: 10 min per iteration
+ * # Threads: 1 thread, will synchronize iterations
+ * # Benchmark mode: Throughput, ops/time
+ * # Benchmark: org.apache.skywalking.apm.util.StringFormatGroupTest.testMatch
+ *
+ * # Run progress: 50.00% complete, ETA 00:00:50
+ * # Fork: 1 of 1
+ * Iteration 1: 1137158.205 ops/s
+ * Iteration 2: 1192936.161 ops/s
+ * Iteration 3: 1218773.403 ops/s
+ * Iteration 4: 1222966.452 ops/s
+ * Iteration 5: 1235609.354 ops/s
+ *
+ *
+ * Result "org.apache.skywalking.apm.util.StringFormatGroupTest.testMatch":
+ * 1201488.715 ±(99.9%) 150813.461 ops/s [Average]
+ * (min, avg, max) = (1137158.205, 1201488.715, 1235609.354), stdev = 39165.777
+ * CI (99.9%): [1050675.254, 1352302.176] (assumes normal distribution)
+ *
+ *
+ * # Run complete. Total time: 00:01:41
+ *
+ * REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on
+ * why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial
+ * experiments, perform baseline and negative tests that provide experimental control, make sure
+ * the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts.
+ * Do not assume the numbers tell you what you want them to tell.
+ *
+ * Benchmark Mode Cnt Score Error Units
+ * StringFormatGroupTest.test100Rule thrpt 5 35490.540 ± 8345.368 ops/s
+ * StringFormatGroupTest.testMatch thrpt 5 1201488.715 ± 150813.461 ops/s
+ *
+ */
+}
diff --git a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/EndpointNameFormater.java b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/EndpointNameFormater.java
new file mode 100644
index 0000000..0b88705
--- /dev/null
+++ b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/EndpointNameFormater.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.aop.server.receiver.mesh;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.skywalking.apm.util.StringFormatGroup;
+import org.apache.skywalking.oap.server.library.util.ResourceUtils;
+import org.slf4j.*;
+
+/**
+ * @author wusheng
+ */
+public class EndpointNameFormater {
+ private static final Logger logger = LoggerFactory.getLogger(EndpointNameFormater.class);
+ private static Map<String, StringFormatGroup> ALL_RULES = new ConcurrentHashMap<>();
+
+ private static void init(String service) {
+ if (ALL_RULES.containsKey(service)) {
+ return;
+ }
+ StringFormatGroup endpointRule = new StringFormatGroup();
+ Properties properties = new Properties();
+ try {
+ InputStream stream = ResourceUtils.class.getClassLoader().getResourceAsStream(service + "_endpoint_naming_rules.properties");
+ if (stream == null) {
+ logger.info("{}_endpoint_naming_rules.properties not found. Try to find global endpoint rule file.", service);
+ stream = ResourceUtils.class.getClassLoader().getResourceAsStream("endpoint_naming_rules.properties");
+ }
+
+ if (stream == null) {
+ logger.info("endpoint_naming_rules.properties not found. No endpoint name setup.");
+ } else {
+ properties.load(stream);
+ properties.forEach((key, value) -> {
+ endpointRule.addRule((String)key, (String)value);
+ });
+ }
+ } catch (IOException e) {
+ logger.info("{}_endpoint_rules.properties not found. No endpoint name setup.", service);
+ }
+
+ ALL_RULES.put(service, endpointRule);
+ }
+
+ public static StringFormatGroup.FormatResult format(String service, String endpointName) {
+ init(service);
+ return ALL_RULES.get(service).format(endpointName);
+ }
+}
diff --git a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java
index b0c1326..06a59d3 100644
--- a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java
+++ b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java
@@ -21,7 +21,8 @@ package org.apache.skywalking.aop.server.receiver.mesh;
import java.util.Objects;
import org.apache.logging.log4j.util.Strings;
import org.apache.skywalking.apm.network.servicemesh.*;
-import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.apm.util.StringFormatGroup;
+import org.apache.skywalking.oap.server.core.*;
import org.apache.skywalking.oap.server.core.cache.*;
import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
import org.apache.skywalking.oap.server.core.register.service.*;
@@ -60,6 +61,13 @@ public class TelemetryDataDispatcher {
}
public static void preProcess(ServiceMeshMetric data) {
+ String service = data.getDestServiceId() == Const.NONE ? data.getDestServiceName() :
+ SERVICE_CACHE.get(data.getDestServiceId()).getName();
+ StringFormatGroup.FormatResult formatResult = EndpointNameFormater.format(service, data.getEndpoint());
+ if (formatResult.isMatch()) {
+ data = data.toBuilder().setEndpoint(formatResult.getName()).build();
+ }
+
ServiceMeshMetricDataDecorator decorator = new ServiceMeshMetricDataDecorator(data);
if (decorator.tryMetaDataRegister()) {
TelemetryDataDispatcher.doDispatch(decorator);
diff --git a/oap-server/server-starter/src/main/resources/endpoint_naming_rules.properties b/oap-server/server-starter/src/main/resources/endpoint_naming_rules.properties
new file mode 100644
index 0000000..aa5cb62
--- /dev/null
+++ b/oap-server/server-starter/src/main/resources/endpoint_naming_rules.properties
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+istio.mixer.v1.Mixer=.*/istio.mixer.v1.Mixer/.*
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index cf483b0..0c3ebb9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,6 +143,7 @@
<coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
<maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
<jacoco-maven-plugin.version>0.8.3</jacoco-maven-plugin.version>
+ <jmh.version>1.21</jmh.version>
</properties>
<dependencies>
@@ -166,6 +167,19 @@
<artifactId>powermock-api-mockito</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<dependencyManagement>