You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/06/16 22:45:01 UTC
[ambari] branch trunk updated: AMBARI-24129. Log Search - Logs
histogram query is not filtered by include/exclude fields. (#1562)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new d5e8fbe AMBARI-24129. Log Search - Logs histogram query is not filtered by include/exclude fields. (#1562)
d5e8fbe is described below
commit d5e8fbe11c2e4c3615bd2553f94e2dc577c515fe
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Sun Jun 17 00:44:35 2018 +0200
AMBARI-24129. Log Search - Logs histogram query is not filtered by include/exclude fields. (#1562)
* AMBARI-24129. Log Search - Logs histogram query is not filtered by include/exclude fields.
* AMBARI-24129. Fix unit tests
---
.../AbstractOperationHolderConverter.java | 54 ++++++++++++++++++++++
.../AuditBarGraphRequestQueryConverter.java | 2 +
...viceLogLevelDateRangeRequestQueryConverter.java | 2 +
.../AuditBarGraphRequestQueryConverterTest.java | 7 +--
...LogLevelDateRangeRequestQueryConverterTest.java | 3 +-
5 files changed, 63 insertions(+), 5 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
index 7fab149..5e7f440 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
@@ -140,6 +140,23 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
return query;
}
+ public SolrQuery addIncludeFieldValues(SolrQuery query, String fieldValuesMapStr) {
+ if (StringUtils.isNotEmpty(fieldValuesMapStr)) {
+ List<Map<String, String>> criterias = new Gson().fromJson(fieldValuesMapStr,
+ new TypeToken<List<HashMap<String, String>>>(){}.getType());
+ for (Map<String, String> criteriaMap : criterias) {
+ for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
+ if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
+ addLogMessageFilter(query, fieldEntry.getValue(), false);
+ } else {
+ query.addFilterQuery(String.format("%s:%s", fieldEntry.getKey(), escapeNonLogMessageField(fieldEntry)));
+ }
+ }
+ }
+ }
+ return query;
+ }
+
public Query addExcludeFieldValues(Query query, String fieldValuesMapStr) {
if (StringUtils.isNotEmpty(fieldValuesMapStr)) {
List<Map<String, String>> criterias = new Gson().fromJson(fieldValuesMapStr,
@@ -157,6 +174,23 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
return query;
}
+ public SolrQuery addExcludeFieldValues(SolrQuery query, String fieldValuesMapStr) {
+ if (StringUtils.isNotEmpty(fieldValuesMapStr)) {
+ List<Map<String, String>> criterias = new Gson().fromJson(fieldValuesMapStr,
+ new TypeToken<List<HashMap<String, String>>>(){}.getType());
+ for (Map<String, String> criteriaMap : criterias) {
+ for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
+ if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
+ addLogMessageFilter(query, fieldEntry.getValue(), true);
+ } else {
+ query.addFilterQuery(String.format("-%s:%s", fieldEntry.getKey(), escapeNonLogMessageField(fieldEntry)));
+ }
+ }
+ }
+ }
+ return query;
+ }
+
public SolrQuery addListFilterToSolrQuery(SolrQuery solrQuery, String fieldName, String fieldValue) {
return SolrUtil.addListFilterToSolrQuery(solrQuery, fieldName, fieldValue);
}
@@ -182,6 +216,26 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
}
}
+ private void addLogMessageFilter(SolrQuery query, String value, boolean negate) {
+ StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"');
+ String negateToken = negate ? "-" : "";
+ for (String token : tokenizer.getTokenArray()) {
+ token = token.trim();
+ if (token.contains(" ") || !token.startsWith("*") && !token.endsWith("*")) {
+ query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(token)));
+ } else if (token.startsWith("*") && token.endsWith("*")) {
+ String plainToken = StringUtils.substring(token, 1, -1);
+ query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
+ } else if (token.startsWith("*") && !token.endsWith("*")) {
+ String plainToken = StringUtils.substring(token, 1);
+ query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
+ } else if (!token.startsWith("*") && token.endsWith("*")) {
+ String plainToken = StringUtils.substring(token, 0, -1);
+ query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
+ }
+ }
+ }
+
private String escapeNonLogMessageField(Map.Entry<String, String> fieldEntry) {
Map<String, String> schemaFieldNameMap = solrSchemaFieldDao.getSchemaFieldNameMap(getLogType());
Map<String, String> schemaFieldTypeMap = solrSchemaFieldDao.getSchemaFieldTypeMap(getLogType());
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
index beb8326..6a6765f 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
@@ -58,6 +58,8 @@ public class AuditBarGraphRequestQueryConverter extends AbstractDateRangeFacetQu
addInFiltersIfNotNullAndEnabled(query, request.getUserList(),
SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
StringUtils.isNotBlank(request.getUserList()));
+ addIncludeFieldValues(query, request.getIncludeQuery());
+ addExcludeFieldValues(query, request.getExcludeQuery());
return query;
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
index f4c0206..db9c60e 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
@@ -61,6 +61,8 @@ public class ServiceLogLevelDateRangeRequestQueryConverter extends AbstractDateR
}
addListFilterToSolrQuery(solrQuery, CLUSTER, request.getClusters());
addListFilterToSolrQuery(solrQuery, COMPONENT, request.getMustBe());
+ addIncludeFieldValues(solrQuery, request.getIncludeQuery());
+ addExcludeFieldValues(solrQuery, request.getExcludeQuery());
return solrQuery;
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
index ba8ae42..96e0c82 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
@@ -45,9 +45,10 @@ public class AuditBarGraphRequestQueryConverterTest extends AbstractRequestConve
request.setUnit("+1HOUR");
// THEN
SolrQuery query = underTest.convert(request);
- assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Drepo&facet.mincount=1&facet.limit=-1&facet.sort=index" +
- "&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR" +
- "&rows=0&start=0&fq=cluster%3Acl1&fq=reqUser%3A%28joe+OR+steven%29",
+ assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Drepo&facet.mincount=1&facet.limit=-1" +
+ "&facet.sort=index&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z" +
+ "&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR&rows=0&start=0" +
+ "&fq=cluster%3Acl1&fq=reqUser%3A%28joe+OR+steven%29&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage",
query.toQueryString());
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
index f30a03d..026d273 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
@@ -47,8 +47,7 @@ public class ServiceLogLevelDateRangeRequestQueryConverterTest extends AbstractR
// THEN
assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Dlevel&facet.mincount=1&facet.limit=-1" +
"&facet.sort=index&facet.range=%7B%21tag%3Dr1%7Dlogtime&f.logtime.facet.range.start=2016-09-13T22%3A00%3A01.000Z" +
- "&f.logtime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.logtime.facet.range.gap=%2B1HOUR&rows=0&start=0" +
- "&fq=level%3A%28WARN+OR+ERROR+OR+FATAL%29&fq=cluster%3Acl1&fq=type%3A%28logsearch_app+OR+secure_log%29", query.toQueryString());
+ "&f.logtime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.logtime.facet.range.gap=%2B1HOUR&rows=0&start=0&fq=level%3A%28WARN+OR+ERROR+OR+FATAL%29&fq=cluster%3Acl1&fq=type%3A%28logsearch_app+OR+secure_log%29&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage", query.toQueryString());
}
@Test
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.