You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2020/10/10 08:41:35 UTC
[skywalking] branch master updated: Add E2E test for python Kafka
report protocol (#5640)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 3981538 Add E2E test for python Kafka report protocol (#5640)
3981538 is described below
commit 39815388a310319424071bd1e618bbc01f8a53cb
Author: Humbertzhang <50...@qq.com>
AuthorDate: Sat Oct 10 16:41:18 2020 +0800
Add E2E test for python Kafka report protocol (#5640)
---
test/e2e/e2e-test/docker/python/Dockerfile.python | 4 +-
test/e2e/e2e-test/docker/python/consumer.py | 6 ++
test/e2e/e2e-test/docker/python/docker-compose.yml | 92 ++++++++++++++++++++++
.../python/{consumer.py => provider-kafka.py} | 22 +++---
.../java/org/apache/skywalking/e2e/PythonE2E.java | 9 ++-
.../{services.yml => provider-kafka-endpoints.yml} | 8 +-
.../{services.yml => provider-kafka-instances.yml} | 8 +-
.../test/resources/expected/python/services.yml | 2 +
.../src/test/resources/expected/python/topo.yml | 10 +++
.../src/test/resources/expected/python/traces.yml | 8 ++
10 files changed, 140 insertions(+), 29 deletions(-)
diff --git a/test/e2e/e2e-test/docker/python/Dockerfile.python b/test/e2e/e2e-test/docker/python/Dockerfile.python
index cdd675c..4d98638 100644
--- a/test/e2e/e2e-test/docker/python/Dockerfile.python
+++ b/test/e2e/e2e-test/docker/python/Dockerfile.python
@@ -15,7 +15,7 @@
FROM python:3.7
-ENV COMMIT_HASH=2056144925a574d04c4a90914a82b8efce2375be
+ENV COMMIT_HASH=50388c55428d742d73d9733278f04173585de80d
WORKDIR /app
@@ -24,6 +24,8 @@ RUN git clone https://github.com/apache/skywalking-python.git $(pwd)
RUN git reset --hard ${COMMIT_HASH} && git submodule update --init
RUN make setup install
+RUN python3 -m pip install kafka-python
ADD ./consumer.py /consumer.py
ADD ./provider.py /provider.py
+ADD ./provider-kafka.py /provider-kafka.py
diff --git a/test/e2e/e2e-test/docker/python/consumer.py b/test/e2e/e2e-test/docker/python/consumer.py
index 67a7be4..4a611c0 100644
--- a/test/e2e/e2e-test/docker/python/consumer.py
+++ b/test/e2e/e2e-test/docker/python/consumer.py
@@ -43,6 +43,12 @@ if __name__ == '__main__':
with request.urlopen(req, data):
self.wfile.write(data)
+ req2 = request.Request("http://provider-kafka:9089/users")
+ req2.add_header('Content-Type', 'application/json; charset=utf-8')
+ req2.add_header('Content-Length', str(len(data)))
+ with request.urlopen(req2, data):
+ self.wfile.write(data)
+
PORT = 9090
Handler = SimpleHTTPRequestHandler
diff --git a/test/e2e/e2e-test/docker/python/docker-compose.yml b/test/e2e/e2e-test/docker/python/docker-compose.yml
index 68eaf03..6fff913 100644
--- a/test/e2e/e2e-test/docker/python/docker-compose.yml
+++ b/test/e2e/e2e-test/docker/python/docker-compose.yml
@@ -16,10 +16,77 @@
version: '2.1'
services:
+ zookeeper:
+ image: zookeeper:3.4
+ hostname: zookeeper
+ expose:
+ - 2181
+ networks:
+ - e2e
+ environment:
+ - ALLOW_ANONYMOUS_LOGIN=yes
+ healthcheck:
+ test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+
+ broker-a:
+ image: bitnami/kafka:2.4.1
+ hostname: broker-a
+ expose:
+ - 9092
+ networks:
+ - e2e
+ environment:
+ - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+ - KAFKA_BROKER_ID=10
+ - ALLOW_PLAINTEXT_LISTENER=yes
+ depends_on:
+ zookeeper:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+
+ broker-b:
+ image: bitnami/kafka:2.4.1
+ hostname: broker-b
+ expose:
+ - 9092
+ networks:
+ - e2e
+ environment:
+ - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+ - KAFKA_BROKER_ID=24
+ - ALLOW_PLAINTEXT_LISTENER=yes
+ depends_on:
+ zookeeper:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+
oap:
extends:
file: ../base-compose.yml
service: oap
+ environment:
+ SW_PROMETHEUS_FETCHER_ACTIVE: "true"
+ SW_TELEMETRY: prometheus
+ SW_KAFKA_FETCHER: default
+ SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092
+ SW_KAFKA_FETCHER_PARTITIONS: 2
+ SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1
+ depends_on:
+ broker-a:
+ condition: service_healthy
+ broker-b:
+ condition: service_healthy
ui:
extends:
@@ -62,6 +129,29 @@ services:
provider:
condition: service_healthy
+ provider-kafka:
+ build:
+ context: .
+ dockerfile: Dockerfile.python
+ networks:
+ - e2e
+ expose:
+ - 9089
+ environment:
+ SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800
+ SW_AGENT_INSTANCE: provider-kafka-instance
+ SW_KAFKA_REPORTER_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092
+ depends_on:
+ oap:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9089"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ entrypoint: ['python3', '/provider-kafka.py']
+
+
consumer:
build:
context: .
@@ -79,6 +169,8 @@ services:
condition: service_healthy
medium:
condition: service_healthy
+ provider-kafka:
+ condition: service_healthy
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090"]
interval: 5s
diff --git a/test/e2e/e2e-test/docker/python/consumer.py b/test/e2e/e2e-test/docker/python/provider-kafka.py
similarity index 69%
copy from test/e2e/e2e-test/docker/python/consumer.py
copy to test/e2e/e2e-test/docker/python/provider-kafka.py
index 67a7be4..75084dd 100644
--- a/test/e2e/e2e-test/docker/python/consumer.py
+++ b/test/e2e/e2e-test/docker/python/provider-kafka.py
@@ -15,35 +15,31 @@
# limitations under the License.
#
-import urllib.parse
+import time
+
from urllib import request
from skywalking import agent, config
if __name__ == '__main__':
- config.service_name = 'consumer'
+ config.service_name = 'provider-kafka'
config.logging_level = 'DEBUG'
- config.protocol = 'http'
- config.collector_address = 'http://oap:12800'
+ config.protocol = "kafka"
agent.start()
import socketserver
from http.server import BaseHTTPRequestHandler
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
+
def do_POST(self):
+ time.sleep(0.15)
self.send_response(200)
- self.send_header('Content-Type', 'application/json; charset=utf-8')
+ self.send_header('Content-Type', 'application/json')
self.end_headers()
+ self.wfile.write('{"name": "whatever"}'.encode('ascii'))
- data = '{"name": "whatever"}'.encode('utf8')
- req = request.Request('http://medium:9092/users')
- req.add_header('Content-Type', 'application/json; charset=utf-8')
- req.add_header('Content-Length', str(len(data)))
- with request.urlopen(req, data):
- self.wfile.write(data)
-
- PORT = 9090
+ PORT = 9089
Handler = SimpleHTTPRequestHandler
with socketserver.TCPServer(("", PORT), Handler) as httpd:
diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java
index 254fc4e..6a0d08d 100644
--- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java
+++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java
@@ -18,7 +18,6 @@
package org.apache.skywalking.e2e;
-import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
import org.apache.skywalking.e2e.annotation.DockerCompose;
@@ -55,6 +54,9 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.DockerComposeContainer;
+import java.util.List;
+import java.util.stream.Collectors;
+
import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
@@ -95,8 +97,8 @@ public class PythonE2E extends SkyWalkingTestAdapter {
@RetryableTest
void services() throws Exception {
- final List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
-
+ List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
+ services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList());
LOGGER.info("services: {}", services);
load("expected/python/services.yml").as(ServicesMatcher.class).verify(services);
@@ -105,6 +107,7 @@ public class PythonE2E extends SkyWalkingTestAdapter {
if ("Your_ApplicationName".equals(service.getLabel())) {
continue;
}
+
LOGGER.info("verifying service instances: {}", service);
verifyServiceMetrics(service);
diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml
similarity index 87%
copy from test/e2e/e2e-test/src/test/resources/expected/python/services.yml
copy to test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml
index 5aea577..9a5a477 100644
--- a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml
+++ b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml
@@ -13,10 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-services:
+endpoints:
- key: not null
- label: provider
- - key: not null
- label: consumer
- - key: not null
- label: Your_ApplicationName
+ label: /users
diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml
similarity index 87%
copy from test/e2e/e2e-test/src/test/resources/expected/python/services.yml
copy to test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml
index 5aea577..7de572a 100644
--- a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml
+++ b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml
@@ -13,10 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-services:
+instances:
- key: not null
- label: provider
- - key: not null
- label: consumer
- - key: not null
- label: Your_ApplicationName
+ label: not null
diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml b/test/e2e/e2e-test/src/test/resources/expected/python/services.yml
index 5aea577..d2404d1 100644
--- a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml
+++ b/test/e2e/e2e-test/src/test/resources/expected/python/services.yml
@@ -17,6 +17,8 @@ services:
- key: not null
label: provider
- key: not null
+ label: provider-kafka
+ - key: not null
label: consumer
- key: not null
label: Your_ApplicationName
diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml b/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml
index efdea72..d9f12ef 100644
--- a/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml
+++ b/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml
@@ -27,6 +27,10 @@ nodes:
type: Python
isReal: true
- id: not null
+ name: provider-kafka
+ type: Python
+ isReal: true
+ - id: not null
name: Your_ApplicationName
type: Tomcat
isReal: true
@@ -43,6 +47,12 @@ calls:
- SERVER
target: ${Your_ApplicationName[0]}
- id: not null
+ source: ${consumer[0]}
+ detectPoints:
+ - CLIENT
+ - SERVER
+ target: ${provider-kafka[0]}
+ - id: not null
source: ${Your_ApplicationName[0]}
detectPoints:
- CLIENT
diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml b/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml
index 6fc49ba..a26eb66 100644
--- a/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml
+++ b/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml
@@ -24,6 +24,14 @@ traces:
- not null
- key: not null
endpointNames:
+ - /users
+ duration: ge 0
+ start: gt 0
+ isError: false
+ traceIds:
+ - not null
+ - key: not null
+ endpointNames:
- /test
duration: ge 0
start: gt 0