You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/01/31 15:35:10 UTC
[skywalking] branch log-es-query created (now e1efb3a)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a change to branch log-es-query
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at e1efb3a Multiple exclude and include query conditions don't work as expected.
This branch includes the following new commits:
new e1efb3a Multiple exclude and include query conditions don't work as expected.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: Multiple exclude and include query conditions
don't work as expected.
Posted by wu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch log-es-query
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e1efb3a75df263c21c7d1dc94631f97456781d68
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Sun Jan 31 23:34:39 2021 +0800
Multiple exclude and include query conditions don't work as expected.
---
.../plugin/elasticsearch/query/LogQueryEsDAO.java | 52 +++++++++++----------
.../elasticsearch7/query/LogQueryEs7DAO.java | 53 +++++++++++-----------
2 files changed, 54 insertions(+), 51 deletions(-)
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java
index 7483ec0..72b4f8d 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java
@@ -20,7 +20,6 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query;
import java.io.IOException;
import java.util.List;
-import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord;
import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
@@ -81,14 +80,13 @@ public class LogQueryEsDAO extends EsDAO implements ILogQueryDAO {
mustQueryList.add(QueryBuilders.rangeQuery(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB));
}
if (isNotEmpty(serviceId)) {
- boolQueryBuilder.must().add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_ID, serviceId));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_ID, serviceId));
}
if (isNotEmpty(serviceInstanceId)) {
- boolQueryBuilder.must()
- .add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_INSTANCE_ID, serviceInstanceId));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_INSTANCE_ID, serviceInstanceId));
}
if (isNotEmpty(endpointId)) {
- boolQueryBuilder.must().add(QueryBuilders.termQuery(AbstractLogRecord.ENDPOINT_ID, endpointId));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.ENDPOINT_ID, endpointId));
}
if (isNotEmpty(endpointName)) {
String matchCName = MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.ENDPOINT_NAME);
@@ -96,38 +94,43 @@ public class LogQueryEsDAO extends EsDAO implements ILogQueryDAO {
}
if (nonNull(relatedTrace)) {
if (isNotEmpty(relatedTrace.getTraceId())) {
- boolQueryBuilder.must()
- .add(QueryBuilders.termQuery(AbstractLogRecord.TRACE_ID, relatedTrace.getTraceId()));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.TRACE_ID, relatedTrace.getTraceId()));
}
if (isNotEmpty(relatedTrace.getSegmentId())) {
- boolQueryBuilder.must().add(
+ mustQueryList.add(
QueryBuilders.termQuery(AbstractLogRecord.TRACE_SEGMENT_ID, relatedTrace.getSegmentId()));
}
if (nonNull(relatedTrace.getSpanId())) {
- boolQueryBuilder.must().add(
- QueryBuilders.termQuery(AbstractLogRecord.SPAN_ID, relatedTrace.getSpanId()));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.SPAN_ID, relatedTrace.getSpanId()));
}
}
if (CollectionUtils.isNotEmpty(tags)) {
- BoolQueryBuilder tagMatchQuery = QueryBuilders.boolQuery();
- tags.forEach(tag -> tagMatchQuery.must(QueryBuilders.termQuery(AbstractLogRecord.TAGS, tag.toString())));
- mustQueryList.add(tagMatchQuery);
+ tags.forEach(tag -> mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.TAGS, tag.toString())));
}
if (CollectionUtils.isNotEmpty(keywordsOfContent)) {
- mustQueryList.add(
- QueryBuilders.matchPhraseQuery(
- MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
- String.join(Const.SPACE, keywordsOfContent)
- ));
+ keywordsOfContent.forEach(
+ content ->
+ mustQueryList.add(
+ QueryBuilders.matchPhraseQuery(
+ MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
+ content
+ )
+ )
+ );
}
if (CollectionUtils.isNotEmpty(excludingKeywordsOfContent)) {
- boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(
- MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
- String.join(Const.SPACE, excludingKeywordsOfContent)
- ));
+ excludingKeywordsOfContent.forEach(
+ content ->
+ boolQueryBuilder.mustNot(
+ QueryBuilders.matchPhraseQuery(
+ MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
+ content
+ )
+ )
+ );
}
sourceBuilder.sort(LogRecord.TIMESTAMP, Order.DES.equals(queryOrder) ? SortOrder.DESC : SortOrder.ASC);
@@ -148,9 +151,8 @@ public class LogQueryEsDAO extends EsDAO implements ILogQueryDAO {
log.setEndpointName((String) searchHit.getSourceAsMap().get(AbstractLogRecord.ENDPOINT_NAME));
log.setTraceId((String) searchHit.getSourceAsMap().get(AbstractLogRecord.TRACE_ID));
log.setTimestamp(searchHit.getSourceAsMap().get(AbstractLogRecord.TIMESTAMP).toString());
- log.setContentType(ContentType.instanceOf(((Number) searchHit.getSourceAsMap()
- .get(
- AbstractLogRecord.CONTENT_TYPE)).intValue()));
+ log.setContentType(ContentType.instanceOf(
+ ((Number) searchHit.getSourceAsMap().get(AbstractLogRecord.CONTENT_TYPE)).intValue()));
log.setContent((String) searchHit.getSourceAsMap().get(AbstractLogRecord.CONTENT));
String dataBinaryBase64 = (String) searchHit.getSourceAsMap().get(AbstractLogRecord.TAGS_RAW_DATA);
if (!Strings.isNullOrEmpty(dataBinaryBase64)) {
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/LogQueryEs7DAO.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/LogQueryEs7DAO.java
index 84cfd03..d66a90d 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/LogQueryEs7DAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/LogQueryEs7DAO.java
@@ -20,7 +20,6 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query;
import java.io.IOException;
import java.util.List;
-import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord;
import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
@@ -82,15 +81,13 @@ public class LogQueryEs7DAO extends EsDAO implements ILogQueryDAO {
}
if (isNotEmpty(serviceId)) {
- boolQueryBuilder.must()
- .add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_ID, serviceId));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_ID, serviceId));
}
if (isNotEmpty(serviceInstanceId)) {
- boolQueryBuilder.must()
- .add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_INSTANCE_ID, serviceInstanceId));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.SERVICE_INSTANCE_ID, serviceInstanceId));
}
if (isNotEmpty(endpointId)) {
- boolQueryBuilder.must().add(QueryBuilders.termQuery(AbstractLogRecord.ENDPOINT_ID, endpointId));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.ENDPOINT_ID, endpointId));
}
if (isNotEmpty(endpointName)) {
String matchCName = MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.ENDPOINT_NAME);
@@ -98,38 +95,43 @@ public class LogQueryEs7DAO extends EsDAO implements ILogQueryDAO {
}
if (nonNull(relatedTrace)) {
if (isNotEmpty(relatedTrace.getTraceId())) {
- boolQueryBuilder.must()
- .add(QueryBuilders.termQuery(AbstractLogRecord.TRACE_ID, relatedTrace.getTraceId()));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.TRACE_ID, relatedTrace.getTraceId()));
}
if (isNotEmpty(relatedTrace.getSegmentId())) {
- boolQueryBuilder.must().add(
+ mustQueryList.add(
QueryBuilders.termQuery(AbstractLogRecord.TRACE_SEGMENT_ID, relatedTrace.getSegmentId()));
}
if (nonNull(relatedTrace.getSpanId())) {
- boolQueryBuilder.must().add(
- QueryBuilders.termQuery(AbstractLogRecord.SPAN_ID, relatedTrace.getSpanId()));
+ mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.SPAN_ID, relatedTrace.getSpanId()));
}
}
if (CollectionUtils.isNotEmpty(tags)) {
- BoolQueryBuilder tagMatchQuery = QueryBuilders.boolQuery();
- tags.forEach(tag -> tagMatchQuery.must(QueryBuilders.termQuery(AbstractLogRecord.TAGS, tag.toString())));
- mustQueryList.add(tagMatchQuery);
+ tags.forEach(tag -> mustQueryList.add(QueryBuilders.termQuery(AbstractLogRecord.TAGS, tag.toString())));
}
if (CollectionUtils.isNotEmpty(keywordsOfContent)) {
- mustQueryList.add(
- QueryBuilders.matchPhraseQuery(
- MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
- String.join(Const.SPACE, keywordsOfContent)
- ));
+ keywordsOfContent.forEach(
+ content ->
+ mustQueryList.add(
+ QueryBuilders.matchPhraseQuery(
+ MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
+ content
+ )
+ )
+ );
}
if (CollectionUtils.isNotEmpty(excludingKeywordsOfContent)) {
- boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(
- MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
- String.join(Const.SPACE, excludingKeywordsOfContent)
- ));
+ excludingKeywordsOfContent.forEach(
+ content ->
+ boolQueryBuilder.mustNot(
+ QueryBuilders.matchPhraseQuery(
+ MatchCNameBuilder.INSTANCE.build(AbstractLogRecord.CONTENT),
+ content
+ )
+ )
+ );
}
sourceBuilder.sort(LogRecord.TIMESTAMP, Order.DES.equals(queryOrder) ? SortOrder.DESC : SortOrder.ASC);
@@ -149,9 +151,8 @@ public class LogQueryEs7DAO extends EsDAO implements ILogQueryDAO {
log.setEndpointName((String) searchHit.getSourceAsMap().get(AbstractLogRecord.ENDPOINT_NAME));
log.setTraceId((String) searchHit.getSourceAsMap().get(AbstractLogRecord.TRACE_ID));
log.setTimestamp(searchHit.getSourceAsMap().get(AbstractLogRecord.TIMESTAMP).toString());
- log.setContentType(ContentType.instanceOf(((Number) searchHit.getSourceAsMap()
- .get(
- AbstractLogRecord.CONTENT_TYPE)).intValue()));
+ log.setContentType(ContentType.instanceOf(
+ ((Number) searchHit.getSourceAsMap().get(AbstractLogRecord.CONTENT_TYPE)).intValue()));
log.setContent((String) searchHit.getSourceAsMap().get(AbstractLogRecord.CONTENT));
String dataBinaryBase64 = (String) searchHit.getSourceAsMap().get(AbstractLogRecord.TAGS_RAW_DATA);
if (!Strings.isNullOrEmpty(dataBinaryBase64)) {