You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2022/06/01 07:32:40 UTC
[skywalking] branch ondemandlog2 updated (411b8f94f9 -> e5ab058597)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch ondemandlog2
in repository https://gitbox.apache.org/repos/asf/skywalking.git
discard 411b8f94f9 Allow querying logs only by instance to simplify frontend query
new e5ab058597 Allow querying logs only by instance to simplify frontend query
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (411b8f94f9)
\
N -- N -- N refs/heads/ondemandlog2 (e5ab058597)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../skywalking/oap/query/graphql/resolver/OndemandLogQuery.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
[skywalking] 01/01: Allow querying logs only by instance to simplify frontend query
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch ondemandlog2
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e5ab058597de33078d9c04f3150c788b72b4f813
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed Jun 1 15:29:53 2022 +0800
Allow querying logs only by instance to simplify frontend query
---
.../analysis/manual/instance/InstanceTraffic.java | 2 +
.../oap/query/graphql/GraphQLQueryProvider.java | 5 +-
.../query/graphql/resolver/OndemandLogQuery.java | 94 +++++++++++++---------
....java => OndemandContainergQueryCondition.java} | 11 +--
.../graphql/type/OndemandLogQueryCondition.java | 2 -
.../src/main/resources/query-protocol | 2 +-
.../main/resources/otel-oc-rules/k8s-instance.yaml | 23 ++++++
7 files changed, 86 insertions(+), 53 deletions(-)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java
index a36c29168d..c413495b81 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java
@@ -175,6 +175,8 @@ public class InstanceTraffic extends Metrics {
}
public static class PropertyUtil {
+ public static final String NAMESPACE = "namespace";
+ public static final String POD = "pod";
public static final String LANGUAGE = "language";
public static final String IPV4 = "ipv4";
public static final String IPV4S = "ipv4s";
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java
index 919fdbb038..f4f694a96e 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java
@@ -78,6 +78,7 @@ public class GraphQLQueryProvider extends ModuleProvider {
@Override
public void prepare() throws ServiceNotProvidedException {
+ final MetadataQueryV2 metadataQueryV2 = new MetadataQueryV2(getManager());
schemaBuilder.file("query-protocol/common.graphqls")
.resolvers(new Query(), new Mutation(), new HealthQuery(getManager()))
.file("query-protocol/metadata.graphqls")
@@ -118,14 +119,14 @@ public class GraphQLQueryProvider extends ModuleProvider {
.file("query-protocol/event.graphqls")
.resolvers(new EventQuery(getManager()))
.file("query-protocol/metadata-v2.graphqls")
- .resolvers(new MetadataQueryV2(getManager()))
+ .resolvers(metadataQueryV2)
.file("query-protocol/ebpf-profiling.graphqls")
.resolvers(new EBPFProcessProfilingQuery(getManager()), new EBPFProcessProfilingMutation(getManager()));
if (config.isEnableOnDemandPodLog()) {
schemaBuilder
.file("query-protocol/ondemand-pod-log.graphqls")
- .resolvers(new OndemandLogQuery());
+ .resolvers(new OndemandLogQuery(metadataQueryV2));
}
schemaBuilder.scalars(ExtendedScalars.GraphQLLong);
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java
index 458572ba16..e768164a46 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java
@@ -25,7 +25,6 @@ import java.io.IOException;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.stream.Collectors;
import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
@@ -34,64 +33,86 @@ import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.skywalking.oap.query.graphql.type.InternalLog;
import org.apache.skywalking.oap.query.graphql.type.LogAdapter;
+import org.apache.skywalking.oap.query.graphql.type.OndemandContainergQueryCondition;
import org.apache.skywalking.oap.query.graphql.type.OndemandLogQueryCondition;
-import org.apache.skywalking.oap.server.core.analysis.IDManager;
+import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic.PropertyUtil;
import org.apache.skywalking.oap.server.core.query.input.Duration;
+import org.apache.skywalking.oap.server.core.query.type.Attribute;
import org.apache.skywalking.oap.server.core.query.type.Log;
import org.apache.skywalking.oap.server.core.query.type.Logs;
+import org.apache.skywalking.oap.server.core.query.type.ServiceInstance;
import org.apache.skywalking.oap.server.library.util.StringUtil;
import graphql.kickstart.tools.GraphQLQueryResolver;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.Configuration;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1Container;
-import io.kubernetes.client.openapi.models.V1Namespace;
-import io.kubernetes.client.openapi.models.V1NamespaceList;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1Pod;
import io.kubernetes.client.openapi.models.V1PodSpec;
import io.kubernetes.client.util.Config;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
+@RequiredArgsConstructor
public class OndemandLogQuery implements GraphQLQueryResolver {
private final Gson gson = new Gson();
private final Type responseType = new TypeToken<Map<String, Object>>() {
}.getType();
private CoreV1Api kApi;
- public List<String> listNamespaces() throws IOException {
- try {
- final V1NamespaceList nsList =
- kApi().listNamespace(null, null, null, null, null, null, null, null, null, null);
- return nsList
- .getItems()
- .stream()
- .map(V1Namespace::getMetadata)
- .filter(Objects::nonNull)
- .map(V1ObjectMeta::getName)
- .collect(Collectors.toList());
- } catch (ApiException e) {
- log.error("Failed to list namespaces from Kubernetes, {}", e.getResponseBody(), e);
+ private final MetadataQueryV2 metadataQuery;
- Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
- String message = responseBody.getOrDefault("message", e.getCode()).toString();
- throw new RuntimeException(message);
- }
+ public List<String> listContainers(final OndemandContainergQueryCondition condition)
+ throws IOException {
+ final ServiceInstance instance =
+ metadataQuery.getInstance(condition.getServiceInstanceId());
+ final Map<String, String> attributesMap = ensureInstanceHasPodProperties(instance);
+ final String ns = attributesMap.get(PropertyUtil.NAMESPACE);
+ final String pod = attributesMap.get(PropertyUtil.POD);
+ return listContainers(ns, pod);
}
- public List<String> listContainers(final OndemandLogQueryCondition condition)
+ public Logs ondemandPodLogs(final OndemandLogQueryCondition condition)
throws IOException {
- final String ns = condition.getNamespace();
- final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition =
- IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId());
- final String instanceName = instanceIDDefinition.getName();
+ final ServiceInstance instance =
+ metadataQuery.getInstance(condition.getServiceInstanceId());
+ final Map<String, String> attributesMap = ensureInstanceHasPodProperties(instance);
+ final String ns = attributesMap.get(PropertyUtil.NAMESPACE);
+ final String pod = attributesMap.get(PropertyUtil.POD);
+ return ondemandPodLogs(ns, pod, condition);
+ }
+ protected Map<String, String> ensureInstanceHasPodProperties(final ServiceInstance instance) {
+ if (instance == null) {
+ throw new IllegalArgumentException("No such instance");
+ }
+ final List<Attribute> attributes = instance.getAttributes();
+ if (attributes == null || attributes.isEmpty()) {
+ throw new IllegalArgumentException(
+ "Service instance has no valid properties to locate the Pod");
+ }
+ final Map<String, String> attributesMap =
+ attributes
+ .stream()
+ .collect(Collectors.toMap(Attribute::getName, Attribute::getValue));
+ if (!attributesMap.containsKey(PropertyUtil.NAMESPACE)
+ || !attributesMap.containsKey(PropertyUtil.POD)) {
+ throw new IllegalArgumentException(
+ "Service instance has no valid properties to locate the Pod");
+ }
+ return attributesMap;
+ }
+
+ protected List<String> listContainers(
+ final String namespace,
+ final String podName) throws IOException {
try {
- final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null);
+ final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null);
final V1PodSpec spec = pod.getSpec();
if (isNull(spec)) {
- throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName));
+ throw new RuntimeException(String.format("No spec: %s:%s", namespace, podName));
}
final List<String> containers = spec.getContainers().stream()
@@ -114,23 +135,20 @@ public class OndemandLogQuery implements GraphQLQueryResolver {
}
}
- public Logs ondemandPodLogs(OndemandLogQueryCondition condition)
- throws IOException {
- final String ns = condition.getNamespace();
- final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition =
- IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId());
- final String instanceName = instanceIDDefinition.getName();
-
+ protected Logs ondemandPodLogs(
+ final String namespace,
+ final String podName,
+ final OndemandLogQueryCondition condition) throws IOException {
try {
- final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null);
+ final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null);
final V1ObjectMeta podMetadata = pod.getMetadata();
if (isNull(podMetadata)) {
throw new RuntimeException(
- String.format("No such instance: %s:%s", ns, instanceName));
+ String.format("No such instance: %s:%s", namespace, podName));
}
final V1PodSpec spec = pod.getSpec();
if (isNull(spec)) {
- throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName));
+ throw new RuntimeException(String.format("No spec: %s:%s", podName, podName));
}
final Duration duration = new Duration();
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java
similarity index 69%
copy from oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
copy to oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java
index d186e1d496..754412e0e6 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java
@@ -19,20 +19,11 @@
package org.apache.skywalking.oap.query.graphql.type;
-import java.util.Collections;
-import java.util.List;
-import org.apache.skywalking.oap.server.core.query.input.Duration;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
-public class OndemandLogQueryCondition {
- private String namespace;
- private String container;
- private String serviceId;
+public class OndemandContainergQueryCondition {
private String serviceInstanceId;
- private Duration duration;
- private List<String> keywordsOfContent = Collections.emptyList();
- private List<String> excludingKeywordsOfContent = Collections.emptyList();
}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
index d186e1d496..43397b2748 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
@@ -28,9 +28,7 @@ import lombok.Setter;
@Getter
@Setter
public class OndemandLogQueryCondition {
- private String namespace;
private String container;
- private String serviceId;
private String serviceInstanceId;
private Duration duration;
private List<String> keywordsOfContent = Collections.emptyList();
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol
index edf5871a3a..faf3a76808 160000
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol
@@ -1 +1 @@
-Subproject commit edf5871a3a2e8f7300eb06175240b56c21072fe9
+Subproject commit faf3a768087825258f64dd782a4dc405dc271cf8
diff --git a/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml b/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml
new file mode 100644
index 0000000000..8d723fbd4b
--- /dev/null
+++ b/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+filter: "{ tags -> tags.job_name in [ 'kubernetes-cadvisor', 'kube-state-metrics' ] }" # The OpenTelemetry job name
+expSuffix: |-
+ service(['cluster' , 'service'], '::', Layer.K8S_SERVICE)
+ .instance(['cluster', 'service'], '::', ['pod'], '', Layer.K8S_SERVICE, { tags -> ['pod': tags.pod, 'namespace': tags.namespace] })
+metricPrefix: k8s_service_instance
+metricsRules:
+ - name: pod_instance_status
+ exp: kube_pod_status_phase.retagByK8sMeta('service' , K8sRetagType.Pod2Service , 'pod' , 'namespace').tagNotEqual('service' , '').sum(['cluster', 'namespace', 'service' , 'pod'])