You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/04/12 12:32:02 UTC
[1/6] ambari git commit: AMBARI-20023. Add delete endpoint for
logsearch (service + audit logs) (oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 688d000d3 -> b172026d6
AMBARI-20023. Add delete endpoint for logsearch (service + audit logs) (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e6df7968
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e6df7968
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e6df7968
Branch: refs/heads/branch-2.5
Commit: e6df7968f62edad5767a0d2396f43201ffdce323
Parents: 688d000
Author: oleewere <ol...@gmail.com>
Authored: Fri Mar 10 14:31:32 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Apr 12 14:31:26 2017 +0200
----------------------------------------------------------------------
.../ambari/logsearch/common/StatusMessage.java | 40 ++++++++++++++++++++
.../ambari/logsearch/conf/ApiDocConfig.java | 4 +-
.../ambari/logsearch/dao/SolrDaoBase.java | 38 ++++++++++++++++---
.../ambari/logsearch/doc/DocConstants.java | 2 +
.../logsearch/manager/AuditLogsManager.java | 8 ++++
.../logsearch/manager/ServiceLogsManager.java | 8 ++++
.../logsearch/rest/AuditLogsResource.java | 9 +++++
.../logsearch/rest/ServiceLogsResource.java | 11 +++++-
8 files changed, 111 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/StatusMessage.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/StatusMessage.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/StatusMessage.java
new file mode 100644
index 0000000..d4567de
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/StatusMessage.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.common;
+
+public class StatusMessage {
+
+ private String status;
+
+ public StatusMessage(String status) {
+ this.status = status;
+ }
+
+ public StatusMessage(int status) {
+ this.status = String.valueOf(status);
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApiDocConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApiDocConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApiDocConfig.java
index 4a8fdea..630e73a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApiDocConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApiDocConfig.java
@@ -25,8 +25,6 @@ import io.swagger.jaxrs.listing.SwaggerSerializers;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import java.net.UnknownHostException;
-
@Configuration
public class ApiDocConfig {
@@ -41,7 +39,7 @@ public class ApiDocConfig {
}
@Bean
- public BeanConfig swaggerConfig() throws UnknownHostException {
+ public BeanConfig swaggerConfig() {
BeanConfig beanConfig = new BeanConfig();
beanConfig.setSchemes(new String[]{"http", "https"});
beanConfig.setBasePath("/api/v1");
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 0568fd7..b30b6ef 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -34,6 +34,8 @@ import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrResponseBase;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.springframework.data.solr.core.DefaultQueryParser;
import org.springframework.data.solr.core.SolrCallback;
import org.springframework.data.solr.core.SolrTemplate;
@@ -64,11 +66,7 @@ public abstract class SolrDaoBase {
solrQuery.remove("event");
try {
QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST);
- if (event != null) {
- LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " +
- solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " +
- queryResponse.getElapsedTime());
- }
+ logSolrEvent(event, solrQuery, queryResponse);
return queryResponse;
} catch (Exception e){
LOG.error("Error during solrQuery=" + e);
@@ -80,6 +78,28 @@ public abstract class SolrDaoBase {
}
}
+ public UpdateResponse deleteByQuery(SolrQuery solrQuery, String event) {
+ SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery);
+ LOG.info("Solr delete query will be processed: " + solrQuery);
+ if (getSolrClient() != null) {
+ try {
+ UpdateResponse updateResponse = getSolrClient().deleteByQuery(solrQuery.getQuery());
+ logSolrEvent(event, solrQuery, updateResponse);
+ return updateResponse;
+ } catch (Exception e) {
+ LOG.error("Error during delete solrQuery=" + e);
+ throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
+ }
+ } else {
+ throw RESTErrorUtil.createRESTException("Solr configuration improper for " + logType.getLabel() + " logs",
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public UpdateResponse deleteByQuery(SolrDataQuery solrDataQuery, String event) {
+ return deleteByQuery(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery), event);
+ }
+
public QueryResponse process(SolrQuery solrQuery) {
return process(solrQuery, null);
}
@@ -109,6 +129,14 @@ public abstract class SolrDaoBase {
return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery), event);
}
+ private void logSolrEvent(String event, SolrQuery solrQuery, SolrResponseBase solrResponseBase) {
+ if (event != null) {
+ LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " +
+ solrQuery + "\nQuery Time Execution :- " + solrResponseBase.getQTime() + " Total Time Elapsed is :- " +
+ solrResponseBase.getElapsedTime());
+ }
+ }
+
public CloudSolrClient getSolrClient() {
return (CloudSolrClient) getSolrTemplate().getSolrClient();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 713c9f6..d57e1ee 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -50,6 +50,7 @@ public class DocConstants {
public class AuditOperationDescriptions {
public static final String GET_AUDIT_SCHEMA_FIELD_LIST_OD = "Get list of schema fields in audit collection";
public static final String GET_AUDIT_LOGS_OD = "Get the list of logs details";
+ public static final String PURGE_AUDIT_LOGS_OD = "Purge service logs based by criteria";
public static final String GET_AUDIT_COMPONENTS_OD = "Get the list of audit components currently active or having data in Solr";
public static final String GET_AUDIT_LINE_GRAPH_DATA_OD = "Get the data required for line graph";
public static final String GET_TOP_AUDIT_RESOURCES_OD = "Get the top audit resource count (grouped by type)";
@@ -76,6 +77,7 @@ public class DocConstants {
public class ServiceOperationDescriptions {
public static final String SEARCH_LOGS_OD = "Searching logs entry";
+ public static final String PURGE_LOGS_OD = "Purge service logs based by criteria";
public static final String GET_HOSTS_OD = "Get the list of service hosts currently active or having data in Solr";
public static final String GET_COMPONENTS_OD = "Get the list of service components currently active or having data in Solr";
public static final String GET_AGGREGATED_INFO_OD = "not required";
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
index 2dc0ef7..2a18415 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
@@ -39,6 +39,7 @@ import freemarker.template.TemplateException;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.StatusMessage;
import org.apache.ambari.logsearch.dao.AuditSolrDao;
import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
@@ -62,6 +63,7 @@ import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.springframework.core.convert.ConversionService;
import org.springframework.data.solr.core.query.SimpleFacetQuery;
import org.springframework.data.solr.core.query.SimpleQuery;
@@ -196,4 +198,10 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
protected AuditLogResponse createLogSearchResponse() {
return new AuditLogResponse();
}
+
+ public StatusMessage deleteLogs(AuditLogRequest request) {
+ SimpleQuery solrQuery = conversionService.convert(request, SimpleQuery.class);
+ UpdateResponse updateResponse = auditSolrDao.deleteByQuery(solrQuery, "/audit/logs");
+ return new StatusMessage(updateResponse.getStatus());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
index f960250..cb1e48f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
@@ -43,6 +43,7 @@ import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.StatusMessage;
import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.model.request.impl.HostLogFilesRequest;
@@ -85,6 +86,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -589,4 +591,10 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
QueryResponse queryResponse = serviceLogsSolrDao.process(facetQuery, "/service/logs/hostlogfiles");
return responseDataGenerator.generateHostLogFilesResponse(queryResponse);
}
+
+ public StatusMessage deleteLogs(ServiceLogRequest request) {
+ SimpleQuery solrQuery = conversionService.convert(request, SimpleQuery.class);
+ UpdateResponse updateResponse = serviceLogsSolrDao.deleteByQuery(solrQuery, "/service/logs");
+ return new StatusMessage(updateResponse.getStatus());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
index d5b6525..2013dc7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
@@ -22,6 +22,7 @@ package org.apache.ambari.logsearch.rest;
import javax.inject.Inject;
import javax.inject.Named;
import javax.ws.rs.BeanParam;
+import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@@ -30,6 +31,7 @@ import javax.ws.rs.core.Response;
import freemarker.template.TemplateException;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.common.StatusMessage;
import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest;
import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest;
@@ -68,6 +70,13 @@ public class AuditLogsResource {
return auditLogsManager.getLogs(auditLogRequest);
}
+ @DELETE
+ @Produces({"application/json"})
+ @ApiOperation(PURGE_AUDIT_LOGS_OD)
+ public StatusMessage deleteAuditLogs(@BeanParam AuditLogRequest auditLogRequest) {
+ return auditLogsManager.deleteLogs(auditLogRequest);
+ }
+
@GET
@Path("/components")
@Produces({"application/json"})
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6df7968/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
index 40247a8..135986f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
@@ -23,6 +23,7 @@ import javax.inject.Named;
import javax.validation.Valid;
import javax.validation.executable.ValidateOnExecution;
import javax.ws.rs.BeanParam;
+import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@@ -31,6 +32,7 @@ import javax.ws.rs.core.Response;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.common.StatusMessage;
import org.apache.ambari.logsearch.model.request.impl.HostLogFilesRequest;
import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest;
import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest;
@@ -67,10 +69,17 @@ public class ServiceLogsResource {
@GET
@Produces({"application/json"})
@ApiOperation(SEARCH_LOGS_OD)
- public ServiceLogResponse searchSolrData(@BeanParam ServiceLogRequest request) {
+ public ServiceLogResponse searchServiceLogs(@BeanParam ServiceLogRequest request) {
return serviceLogsManager.searchLogs(request);
}
+ @DELETE
+ @Produces({"application/json"})
+ @ApiOperation(PURGE_LOGS_OD)
+ public StatusMessage deleteServiceLogs(@BeanParam ServiceLogRequest request) {
+ return serviceLogsManager.deleteLogs(request);
+ }
+
@GET
@Path("/hosts")
@Produces({"application/json"})
[2/6] ambari git commit: AMBARI-20378. Appendum - Logfeeder: add
de-duplication support (oleewere)
Posted by ol...@apache.org.
AMBARI-20378. Appendum - Logfeeder: add de-duplication support (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b172026d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b172026d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b172026d
Branch: refs/heads/branch-2.5
Commit: b172026d66de0b2d24e514f9b0abbe77555ec54b
Parents: 9bc97c4
Author: oleewere <ol...@gmail.com>
Authored: Thu Mar 23 13:09:00 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Apr 12 14:31:27 2017 +0200
----------------------------------------------------------------------
.../LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b172026d/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
index ca14c1f..1ff8ad3 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
@@ -115,8 +115,8 @@
<name>logfeeder.cache.key.field</name>
<value>log_message</value>
<description>
- Key field, which will be used as keys in the Input cache. (by defalt, log_message represets the message part of processed data)<
- /description>
+ Key field, which will be used as keys in the Input cache. (by defalt, log_message represets the message part of processed data)
+ </description>
<display-name>Input cache key field</display-name>
<on-ambari-upgrade add="true"/>
</property>
[3/6] ambari git commit: AMBARI-20300. LogSearch Portal UI Fails on
Last Page selection if logs were deleted in the meantime (Miklos Gergely via
oleewere)
Posted by ol...@apache.org.
AMBARI-20300. LogSearch Portal UI Fails on Last Page selection if logs were deleted in the meantime (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a65c5a05
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a65c5a05
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a65c5a05
Branch: refs/heads/branch-2.5
Commit: a65c5a0564c28f781fe5a3407c0ba4dd9a89a92e
Parents: 76ff5e0
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Fri Mar 10 14:44:15 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Apr 12 14:31:27 2017 +0200
----------------------------------------------------------------------
.../AbstractSearchRequestQueryConverter.java | 31 +++++++++++++++++---
.../logsearch/manager/AuditLogsManager.java | 20 +++++++++++--
.../ambari/logsearch/manager/ManagerBase.java | 11 ++++---
.../logsearch/manager/ServiceLogsManager.java | 13 ++++++--
.../scripts/collections/BaseCollection.js | 4 +--
.../webapp/scripts/views/common/TableLayout.js | 2 +-
6 files changed, 64 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65c5a05/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
index d44b866..d4fc48b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
@@ -18,10 +18,15 @@
*/
package org.apache.ambari.logsearch.converter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.ambari.logsearch.model.request.LastPageParamDefinition;
import org.apache.ambari.logsearch.model.request.impl.CommonSearchRequest;
import org.apache.commons.lang.StringUtils;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
+import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.solr.core.query.Criteria;
import org.springframework.data.solr.core.query.Query;
import org.springframework.data.solr.core.query.SimpleStringCriteria;
@@ -32,14 +37,32 @@ public abstract class AbstractSearchRequestQueryConverter<REQUEST_TYPE extends C
@Override
public QUERY_TYPE convert(REQUEST_TYPE request) {
QUERY_TYPE query = createQuery();
- int page = StringUtils.isNumeric(request.getPage()) ? new Integer(request.getPage()) : 0;
- int pageSize = StringUtils.isNumeric(request.getPageSize()) ? new Integer(request.getPageSize()) : 99999;
- PageRequest pageRequest = new PageRequest(page, pageSize, sort(request));
- query.setPageRequest(pageRequest);
+ addPageRequest(request, query);
Criteria criteria = new SimpleStringCriteria("*:*");
query.addCriteria(criteria);
return extendSolrQuery(request, query);
}
+
+ private void addPageRequest(REQUEST_TYPE request, QUERY_TYPE query) {
+ int page = StringUtils.isNumeric(request.getPage()) ? new Integer(request.getPage()) : 0;
+ int pageSize = StringUtils.isNumeric(request.getPageSize()) ? new Integer(request.getPageSize()) : 99999;
+ Sort sort = sort(request);
+
+ boolean isLastPage = (request instanceof LastPageParamDefinition) ?
+ ((LastPageParamDefinition)request).isLastPage() :
+ false;
+ if (isLastPage) {
+ page = 0;
+ List<Sort.Order> newOrders = new ArrayList<>();
+ for (Sort.Order order : sort) {
+ newOrders.add(new Sort.Order(order.getDirection() == Direction.ASC ? Direction.DESC : Direction.ASC, order.getProperty()));
+ }
+ sort = new Sort(newOrders);
+ }
+
+ PageRequest pageRequest = new PageRequest(page, pageSize, sort);
+ query.setPageRequest(pageRequest);
+ }
public abstract QUERY_TYPE extendSolrQuery(REQUEST_TYPE request, QUERY_TYPE query);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65c5a05/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
index 2a18415..49465d5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
@@ -58,6 +58,7 @@ import org.apache.ambari.logsearch.solr.model.SolrAuditLogData;
import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData;
import org.apache.ambari.logsearch.util.DownloadUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ambari.logsearch.common.VResponse;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
@@ -87,8 +88,23 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
@Inject
private SolrSchemaFieldDao solrSchemaFieldDao;
- public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) {
- return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs");
+ public AuditLogResponse getLogs(AuditLogRequest request) {
+ String event = "/audit/logs";
+ SimpleQuery solrQuery = conversionService.convert(request, SimpleQuery.class);
+ if (request.isLastPage()) {
+ return getLastPage(auditSolrDao, solrQuery, event);
+ } else {
+ AuditLogResponse response = getLogAsPaginationProvided(solrQuery, auditSolrDao, event);
+ if (response.getTotalCount() > 0 && CollectionUtils.isEmpty(response.getLogList())) {
+ request.setLastPage(true);
+ solrQuery = conversionService.convert(request, SimpleQuery.class);
+ AuditLogResponse lastResponse = getLastPage(auditSolrDao, solrQuery, event);
+ if (lastResponse != null){
+ response = lastResponse;
+ }
+ }
+ return response;
+ }
}
private List<LogData> getComponents(AuditComponentRequest request) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65c5a05/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
index 89873f3..6b40cb5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
@@ -26,7 +26,6 @@ import java.util.List;
import org.apache.ambari.logsearch.model.response.LogData;
import org.apache.ambari.logsearch.model.response.LogSearchResponse;
import org.apache.ambari.logsearch.dao.SolrDaoBase;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.QueryResponse;
@@ -45,9 +44,9 @@ public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE
protected SEARCH_RESPONSE getLastPage(SolrDaoBase solrDoaBase, SimpleQuery lastPageQuery, String event) {
int maxRows = lastPageQuery.getRows();
SEARCH_RESPONSE logResponse = getLogAsPaginationProvided(lastPageQuery, solrDoaBase, event);
- Long totalLogs = solrDoaBase.count(lastPageQuery);
- int startIndex = Integer.parseInt("" + ((totalLogs / maxRows) * maxRows));
- int numberOfLogsOnLastPage = Integer.parseInt("" + (totalLogs - startIndex));
+ Long totalLogs = logResponse.getTotalCount();
+ int startIndex = (int)(totalLogs - totalLogs % maxRows);
+ int numberOfLogsOnLastPage = (int)(totalLogs - startIndex);
logResponse.setStartIndex(startIndex);
logResponse.setTotalCount(totalLogs);
logResponse.setPageSize(maxRows);
@@ -75,11 +74,11 @@ public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE
QueryResponse response = solrDaoBase.process(solrQuery, event);
SEARCH_RESPONSE logResponse = createLogSearchResponse();
SolrDocumentList docList = response.getResults();
+ logResponse.setTotalCount(docList.getNumFound());
List<LOG_DATA_TYPE> serviceLogDataList = convertToSolrBeans(response);
- if (CollectionUtils.isNotEmpty(docList)) {
+ if (!docList.isEmpty()) {
logResponse.setLogList(serviceLogDataList);
logResponse.setStartIndex((int) docList.getStart());
- logResponse.setTotalCount(docList.getNumFound());
Integer rowNumber = solrQuery.getRows();
if (rowNumber == null) {
logger.error("No RowNumber was set in solrQuery");
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65c5a05/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
index 157335a..9ce209b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
@@ -139,12 +139,21 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
} else if (isLastPage) {
ServiceLogResponse logResponse = getLastPage(serviceLogsSolrDao, solrQuery, event);
- if(logResponse == null){
+ if (logResponse == null){
logResponse = new ServiceLogResponse();
}
return logResponse;
} else {
- return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao, event);
+ ServiceLogResponse response = getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao, event);
+ if (response.getTotalCount() > 0 && CollectionUtils.isEmpty(response.getLogList())) {
+ request.setLastPage(true);
+ solrQuery = conversionService.convert(request, SimpleQuery.class);
+ ServiceLogResponse lastResponse = getLastPage(serviceLogsSolrDao, solrQuery, event);
+ if (lastResponse != null){
+ response = lastResponse;
+ }
+ }
+ return response;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65c5a05/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js
index c175397..82fe6a9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js
@@ -129,9 +129,9 @@ define(['require',
},
getPage : function(index, options){
if(index === "last"){
- this.queryParams.isLastPage = true;
+ this.queryParams.lastPage = true;
}else{
- delete this.queryParams.isLastPage;
+ delete this.queryParams.lastPage;
}
var fn = Backbone.PageableCollection.prototype.getPage;
fn.apply(this,arguments);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a65c5a05/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/TableLayout.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/TableLayout.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/TableLayout.js
index 1066510..b2f0871 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/TableLayout.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/TableLayout.js
@@ -392,7 +392,7 @@ define(['require',
this.collection.state.pageSize = parseInt(pagesize,10);
this.collection.state.currentPage = this.collection.state.firstPage;
- delete this.collection.queryParams.isLastPage;
+ delete this.collection.queryParams.lastPage;
this.collection.fetch({
sort: false,
reset: true,
[4/6] ambari git commit: AMBARI-20117. Log Search: add global cluster
filter for every query (oleewere)
Posted by ol...@apache.org.
AMBARI-20117. Log Search: add global cluster filter for every query (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/edcc29f1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/edcc29f1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/edcc29f1
Branch: refs/heads/branch-2.5
Commit: edcc29f1b373b690f37fd7f2cd1b7f618a5cf4c7
Parents: e6df796
Author: oleewere <ol...@gmail.com>
Authored: Fri Mar 10 14:38:37 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Apr 12 14:31:27 2017 +0200
----------------------------------------------------------------------
.../logsearch/common/LogSearchConstants.java | 1 +
.../AbstractLogRequestFacetQueryConverter.java | 2 ++
.../AbstractLogRequestQueryConverter.java | 3 ++
.../AbstractOperationHolderConverter.java | 5 +++
.../AuditBarGraphRequestQueryConverter.java | 9 ++++++
.../HostLogFilesRequestQueryConverter.java | 4 +--
...eLogLevelDateRangeRequestQueryConverter.java | 12 ++------
.../UserConfigRequestQueryConverter.java | 3 ++
.../ambari/logsearch/doc/DocConstants.java | 1 +
.../logsearch/manager/ServiceLogsManager.java | 30 +++++++++++-------
.../model/request/ClustersParamDefinition.java | 32 ++++++++++++++++++++
.../logsearch/model/request/SearchRequest.java | 5 +--
.../model/request/impl/CommonSearchRequest.java | 17 ++++++++++-
.../model/request/impl/HostLogFilesRequest.java | 15 +++++++++
.../logsearch/rest/ServiceLogsResource.java | 19 +++++++-----
.../apache/ambari/logsearch/util/SolrUtil.java | 13 ++++++++
.../converter/AbstractRequestConverterTest.java | 1 +
.../AuditBarGraphRequestQueryConverterTest.java | 3 +-
...AuditComponentRequestQueryConverterTest.java | 2 +-
.../converter/AuditLogRequestConverterTest.java | 2 +-
...ditServiceLoadRequestQueryConverterTest.java | 6 ++--
...BaseServiceLogRequestQueryConverterTest.java | 6 ++--
.../FieldAuditLogRequestQueryConverterTest.java | 6 ++--
.../ServiceLogAnyGraphRequestConverterTest.java | 7 +++--
...ComponentLevelRequestQueryConverterTest.java | 4 +--
...ComponentRequestFacetQueryConverterTest.java | 2 +-
...eLogLevelCountRequestQueryConverterTest.java | 6 ++--
...LevelDateRangeRequestQueryConverterTest.java | 8 ++---
...ceLogTreeRequestFacetQueryConverterTest.java | 4 +--
...ceLogTruncatedRequestQueryConverterTest.java | 2 +-
.../UserConfigRequestQueryConverterTest.java | 3 +-
.../UserExportRequestQueryConverterTest.java | 1 +
32 files changed, 170 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
index a260ed0..3a74918 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
@@ -71,6 +71,7 @@ public class LogSearchConstants {
public static final String REQUEST_PARAM_STACK_BY = "stackBy";
public static final String REQUEST_PARAM_UNIT = "unit";
public static final String REQUEST_PARAM_TOP = "top";
+ public static final String REQUEST_PARAM_CLUSTER_NAMES = "clusters";
public static final String REQUEST_PARAM_BUNDLE_ID = "bundle_id";
public static final String REQUEST_PARAM_START_INDEX = "startIndex";
public static final String REQUEST_PARAM_PAGE = "page";
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java
index b377968..d5e807b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestFacetQueryConverter.java
@@ -30,6 +30,7 @@ import org.springframework.data.solr.core.query.SimpleStringCriteria;
import java.util.List;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT;
import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT;
@@ -56,6 +57,7 @@ public abstract class AbstractLogRequestFacetQueryConverter<SOURCE extends BaseL
facetQuery.setRows(0);
addComponentFilters(facetQuery, request);
appendFacetQuery(facetQuery, request);
+ addInFilterQuery(facetQuery, CLUSTER, splitValueAsList(request.getClusters(), ","));
return facetQuery;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestQueryConverter.java
index 9b7c9d2..d9f51f4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractLogRequestQueryConverter.java
@@ -22,6 +22,8 @@ import org.apache.ambari.logsearch.model.request.impl.BaseLogRequest;
import org.apache.commons.lang.StringEscapeUtils;
import org.springframework.data.solr.core.query.Query;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
+
public abstract class AbstractLogRequestQueryConverter<REQUEST_TYPE extends BaseLogRequest, QUERY_TYPE extends Query>
extends AbstractSearchRequestQueryConverter<REQUEST_TYPE, QUERY_TYPE> {
@@ -30,6 +32,7 @@ public abstract class AbstractLogRequestQueryConverter<REQUEST_TYPE extends Base
addComponentFilters(request, query);
addIncludeFieldValues(query, StringEscapeUtils.unescapeXml(request.getIncludeQuery()));
addExcludeFieldValues(query, StringEscapeUtils.unescapeXml(request.getExcludeQuery()));
+ addInFilterQuery(query, CLUSTER, splitValueAsList(request.getClusters(), ","));
return extendLogQuery(request, query);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
index b4c6943..c92dd14 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
@@ -27,6 +27,7 @@ import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.text.StrTokenizer;
+import org.apache.solr.client.solrj.SolrQuery;
import org.springframework.data.solr.core.query.Criteria;
import org.springframework.data.solr.core.query.Query;
import org.springframework.data.solr.core.query.SimpleFilterQuery;
@@ -143,6 +144,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
return query;
}
+ public SolrQuery addListFilterToSolrQuery(SolrQuery solrQuery, String fieldName, String fieldValue) {
+ return SolrUtil.addListFilterToSolrQuery(solrQuery, fieldName, fieldValue);
+ }
+
public abstract LogType getLogType();
private void addLogMessageFilter(Query query, String value, boolean negate) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
index 897bb64..06dfae9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
@@ -20,11 +20,13 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
+import org.apache.solr.client.solrj.SolrQuery;
import javax.inject.Named;
import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT;
import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
@Named
public class AuditBarGraphRequestQueryConverter extends AbstractDateRangeFacetQueryConverter<AuditBarGraphRequest> {
@@ -43,4 +45,11 @@ public class AuditBarGraphRequestQueryConverter extends AbstractDateRangeFacetQu
public LogType getLogType() {
return LogType.AUDIT;
}
+
+ @Override
+ public SolrQuery convert(AuditBarGraphRequest request) {
+ SolrQuery query = super.convert(request);
+ addListFilterToSolrQuery(query, CLUSTER, request.getClusters());
+ return query;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java
index d9d9121..e1e7418 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/HostLogFilesRequestQueryConverter.java
@@ -26,6 +26,7 @@ import org.springframework.data.solr.core.query.FacetOptions.FacetSort;
import org.springframework.data.solr.core.query.SimpleFacetQuery;
import org.springframework.data.solr.core.query.SimpleStringCriteria;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.PATH;
@@ -42,14 +43,13 @@ public class HostLogFilesRequestQueryConverter extends AbstractOperationHolderCo
if (StringUtils.isNotEmpty(request.getComponentName())) {
facetQuery.addCriteria(new SimpleStringCriteria(String.format("%s:(%s)", COMPONENT, request.getComponentName())));
}
-
FacetOptions facetOptions = new FacetOptions();
facetOptions.setFacetMinCount(1);
facetOptions.setFacetLimit(-1);
facetOptions.setFacetSort(FacetSort.COUNT);
facetOptions.addFacetOnPivot(COMPONENT, PATH);
facetQuery.setFacetOptions(facetOptions);
-
+ addInFilterQuery(facetQuery, CLUSTER, splitValueAsList(request.getClusters(), ","));
facetQuery.setRows(0);
return facetQuery;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
index b91a67d..46f8f85 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.logsearch.converter;
-import com.google.common.base.Splitter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest;
import org.apache.commons.lang.StringUtils;
@@ -29,6 +28,7 @@ import javax.inject.Named;
import java.util.Arrays;
import java.util.List;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.HOST;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME;
@@ -49,14 +49,7 @@ public class ServiceLogLevelDateRangeRequestQueryConverter extends AbstractDateR
@Override
public SolrQuery convert(ServiceGraphRequest request) {
SolrQuery solrQuery = super.convert(request);
- if (StringUtils.isNotEmpty(request.getLevel())) {
- List<String> levels = Splitter.on(",").splitToList(request.getLevel());
- if (levels.size() > 1) {
- solrQuery.addFilterQuery(String.format("%s:(%s)", LEVEL, StringUtils.join(levels, " OR ")));
- } else {
- solrQuery.addFilterQuery(String.format("%s:%s", LEVEL, levels.get(0)));
- }
- }
+ addListFilterToSolrQuery(solrQuery, LEVEL, request.getLevel());
if (request.getHostList() != null && StringUtils.isEmpty(request.getHostName())) {
List<String> hosts = request.getHostList().length() == 0 ? Arrays.asList("\\-1") : splitValueAsList(request.getHostList(), ",");
if (hosts.size() > 1) {
@@ -65,6 +58,7 @@ public class ServiceLogLevelDateRangeRequestQueryConverter extends AbstractDateR
solrQuery.addFilterQuery(String.format("%s:%s", HOST, hosts.get(0)));
}
}
+ addListFilterToSolrQuery(solrQuery, CLUSTER, request.getClusters());
return solrQuery;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java
index fc75a17..594f23b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverter.java
@@ -28,6 +28,7 @@ import javax.inject.Named;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.FILTER_NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.UserConfigConstants.ROW_TYPE;
@@ -59,6 +60,8 @@ public class UserConfigRequestQueryConverter extends AbstractConverterAware<User
sort.add(sortOrder);
userConfigQuery.setSorts(sort);
+ SolrUtil.addListFilterToSolrQuery(userConfigQuery, CLUSTER, userConfigRequest.getClusters());
+
return userConfigQuery;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index d57e1ee..984e834 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -61,6 +61,7 @@ public class DocConstants {
public class ServiceDescriptions {
public static final String LEVEL_D = "filter for log level";
public static final String BUNDLE_ID = "filter for host";
+ public static final String CLUSTER_D = "filter for clusters (comma separated list)";
public static final String FILE_NAME_D = "File name filter which is supported from browser url";
public static final String HOST_NAME_D = "Host name filter which is supported from browser url";
public static final String COMPONENT_NAME_D = "Component name filter which is supported from browser url";
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
index cb1e48f..157335a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
@@ -34,6 +34,7 @@ import javax.inject.Named;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import freemarker.template.Configuration;
import freemarker.template.Template;
@@ -98,6 +99,7 @@ import org.springframework.data.solr.core.query.SimpleFilterQuery;
import org.springframework.data.solr.core.query.SimpleQuery;
import org.springframework.data.solr.core.query.SimpleStringCriteria;
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ID;
import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.SEQUENCE_ID;
import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.COMPONENT;
@@ -146,12 +148,12 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
}
- public GroupListResponse getHosts() {
- return getFields(HOST, SolrHostLogData.class);
+ public GroupListResponse getHosts(String clusters) {
+ return getFields(HOST, clusters, SolrHostLogData.class);
}
- public GroupListResponse getComponents() {
- return getFields(COMPONENT, SolrComponentTypeLogData.class);
+ public GroupListResponse getComponents(String clusters) {
+ return getFields(COMPONENT, clusters, SolrComponentTypeLogData.class);
}
public GraphDataListResponse getAggregatedInfo(ServiceLogAggregatedInfoRequest request) {
@@ -164,16 +166,21 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
return responseDataGenerator.generateSimpleGraphResponse(response, hierarchy);
}
- public CountDataListResponse getFieldCount(String field) {
- return responseDataGenerator.generateCountResponseByField(serviceLogsSolrDao.process(conversionService.convert(field, SimpleFacetQuery.class)), field);
+ public CountDataListResponse getFieldCount(String field, String clusters) {
+ SimpleFacetQuery facetQuery = conversionService.convert(field, SimpleFacetQuery.class);
+ if (StringUtils.isEmpty(clusters)) {
+ List<String> clusterFilterList = Splitter.on(",").splitToList(clusters);
+ facetQuery.addFilterQuery(new SimpleFilterQuery(new Criteria(CLUSTER).in(clusterFilterList)));
+ }
+ return responseDataGenerator.generateCountResponseByField(serviceLogsSolrDao.process(facetQuery), field);
}
- public CountDataListResponse getComponentsCount() {
- return getFieldCount(COMPONENT);
+ public CountDataListResponse getComponentsCount(String clusters) {
+ return getFieldCount(COMPONENT, clusters);
}
- public CountDataListResponse getHostsCount() {
- return getFieldCount(HOST);
+ public CountDataListResponse getHostsCount(String clusters) {
+ return getFieldCount(HOST, clusters);
}
public NodeListResponse getTreeExtension(ServiceLogHostComponentRequest request) {
@@ -525,9 +532,10 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
}
- private <T extends LogData> GroupListResponse getFields(String field, Class<T> clazz) {
+ private <T extends LogData> GroupListResponse getFields(String field, String clusters, Class<T> clazz) {
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery("*:*");
+ SolrUtil.addListFilterToSolrQuery(solrQuery, CLUSTER, clusters);
GroupListResponse collection = new GroupListResponse();
SolrUtil.setFacetField(solrQuery,
field);
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/ClustersParamDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/ClustersParamDefinition.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/ClustersParamDefinition.java
new file mode 100644
index 0000000..cb15b0a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/ClustersParamDefinition.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.model.request;
+
+import io.swagger.annotations.ApiParam;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.ServiceDescriptions.CLUSTER_D;
+
+public interface ClustersParamDefinition {
+
+ String getClusters();
+
+ @ApiParam(value = CLUSTER_D, name = LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES)
+ void setClusters(String cluster);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/SearchRequest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/SearchRequest.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/SearchRequest.java
index 0015c09..8b275b6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/SearchRequest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/SearchRequest.java
@@ -18,8 +18,5 @@
*/
package org.apache.ambari.logsearch.model.request;
-import org.apache.ambari.logsearch.common.Marker;
-
-@Marker
-public interface SearchRequest {
+public interface SearchRequest extends ClustersParamDefinition {
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
index 2617c51..b763830 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
@@ -24,6 +24,7 @@ import org.apache.ambari.logsearch.model.request.SearchRequest;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
+import javax.annotation.Nullable;
import javax.ws.rs.QueryParam;
public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefinition {
@@ -49,6 +50,10 @@ public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefi
@QueryParam(LogSearchConstants.REQUEST_PARAM_END_TIME)
private String endTime;
+ @Nullable
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES)
+ private String clusters;
+
@Override
public String getStartIndex() {
return startIndex;
@@ -118,7 +123,17 @@ public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefi
public void setEndTime(String endTime) {
this.endTime = endTime;
}
-
+
+ @Override
+ public String getClusters() {
+ return clusters;
+ }
+
+ @Override
+ public void setClusters(String clusters) {
+ this.clusters = clusters;
+ }
+
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/HostLogFilesRequest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/HostLogFilesRequest.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/HostLogFilesRequest.java
index 1c0c068..b2149aa 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/HostLogFilesRequest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/HostLogFilesRequest.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.model.request.impl;
+import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;
import javax.ws.rs.QueryParam;
@@ -34,6 +35,10 @@ public class HostLogFilesRequest implements HostComponentParamDefinition, Search
@QueryParam(LogSearchConstants.REQUEST_PARAM_COMPONENT_NAME)
private String componentName;
+ @Nullable
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES)
+ private String clusters;
+
@Override
public String getHostName() {
return hostName;
@@ -53,4 +58,14 @@ public class HostLogFilesRequest implements HostComponentParamDefinition, Search
public void setComponentName(String componentName) {
this.componentName = componentName;
}
+
+ @Override
+ public String getClusters() {
+ return clusters;
+ }
+
+ @Override
+ public void setClusters(String clusters) {
+ this.clusters = clusters;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
index 135986f..498da69 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.rest;
+import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.validation.Valid;
@@ -27,11 +28,13 @@ import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.StatusMessage;
import org.apache.ambari.logsearch.model.request.impl.HostLogFilesRequest;
import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest;
@@ -84,16 +87,16 @@ public class ServiceLogsResource {
@Path("/hosts")
@Produces({"application/json"})
@ApiOperation(GET_HOSTS_OD)
- public GroupListResponse getHosts() {
- return serviceLogsManager.getHosts();
+ public GroupListResponse getHosts(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) {
+ return serviceLogsManager.getHosts(clusters);
}
@GET
@Path("/components")
@Produces({"application/json"})
@ApiOperation(GET_COMPONENTS_OD)
- public GroupListResponse getComponents() {
- return serviceLogsManager.getComponents();
+ public GroupListResponse getComponents(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) {
+ return serviceLogsManager.getComponents(clusters);
}
@GET
@@ -108,16 +111,16 @@ public class ServiceLogsResource {
@Path("/components/count")
@Produces({"application/json"})
@ApiOperation(GET_COMPONENTS_COUNT_OD)
- public CountDataListResponse getComponentsCount() {
- return serviceLogsManager.getComponentsCount();
+ public CountDataListResponse getComponentsCount(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) {
+ return serviceLogsManager.getComponentsCount(clusters);
}
@GET
@Path("/hosts/count")
@Produces({"application/json"})
@ApiOperation(GET_HOSTS_COUNT_OD)
- public CountDataListResponse getHostsCount() {
- return serviceLogsManager.getHostsCount();
+ public CountDataListResponse getHostsCount(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) {
+ return serviceLogsManager.getHostsCount(clusters);
}
@GET
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
index a3824de..973e304 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Splitter;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
@@ -178,6 +179,18 @@ public class SolrUtil {
return null;
}
}
+
+ public static SolrQuery addListFilterToSolrQuery(SolrQuery solrQuery, String fieldName, String fieldValue) {
+ if (org.apache.commons.lang.StringUtils.isNotEmpty(fieldValue)) {
+ List<String> clusters = Splitter.on(",").splitToList(fieldValue);
+ if (clusters.size() > 1) {
+ solrQuery.addFilterQuery(String.format("%s:(%s)", fieldName, org.apache.commons.lang.StringUtils.join(clusters, " OR ")));
+ } else {
+ solrQuery.addFilterQuery(String.format("%s:%s", fieldName, clusters.get(0)));
+ }
+ }
+ return solrQuery;
+ }
private static Map<String, Object> getFieldTypeInfoMap(String fieldTypeMetaData) {
HashMap<String, Object> fieldTypeMap = JSONUtil.jsonToMapObject(fieldTypeMetaData);
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AbstractRequestConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AbstractRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AbstractRequestConverterTest.java
index 04bd1af..0bff00e 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AbstractRequestConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AbstractRequestConverterTest.java
@@ -37,6 +37,7 @@ public class AbstractRequestConverterTest {
request.setStartIndex("0");
request.setPage("0");
request.setPageSize("25");
+ request.setClusters("cl1");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
index ef12f71..b9e577c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
@@ -44,7 +44,8 @@ public class AuditBarGraphRequestQueryConverterTest extends AbstractRequestConve
// THEN
SolrQuery query = underTest.convert(request);
assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Drepo&facet.mincount=1&facet.limit=-1&facet.sort=index" +
- "&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR&rows=0&start=0",
+ "&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR" +
+ "&rows=0&start=0&fq=cluster%3Acl1",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java
index 0c6e975..7a1b08d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditComponentRequestQueryConverterTest.java
@@ -46,7 +46,7 @@ public class AuditComponentRequestQueryConverterTest extends AbstractRequestConv
SimpleFacetQuery facetQuery = underTest.convert(request);
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(facetQuery);
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B*+TO+*%5D&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.field=repo",
+ assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B*+TO+*%5D&fq=cluster%3Acl1&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.field=repo",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
index 1aae82a..330a036 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
@@ -46,7 +46,7 @@ public class AuditLogRequestConverterTest extends AbstractRequestConverterTest {
SolrQuery queryResult = new DefaultQueryParser().doConstructSolrQuery(simpleQuery);
// THEN
assertEquals("?q=*%3A*&start=0&rows=25&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=evtTime+desc%2Cseq_num+desc",
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=cluster%3Acl1&sort=evtTime+desc%2Cseq_num+desc",
queryResult.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
index 2b629e5..cce9115 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
@@ -43,9 +43,9 @@ public class AuditServiceLoadRequestQueryConverterTest extends AbstractRequestCo
// WHEN
SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage&" +
- "fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
- "&facet=true&facet.mincount=1&facet.limit=10&facet.field=repo", solrQuery.toQueryString());
+ assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29" +
+ "&fq=-repo%3A%28hst_agent+system_message%29&fq=cluster%3Acl1&facet=true&facet.mincount=1&facet.limit=10&facet.field=repo", solrQuery.toQueryString());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
index 89f8eba..da7d79c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
@@ -53,9 +53,9 @@ public class BaseServiceLogRequestQueryConverterTest extends AbstractRequestConv
SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery);
// THEN
assertEquals("?q=*%3A*&start=0&rows=25&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=host%3Alogsearch.com" +
- "&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" +
- "&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc",
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=cluster%3Acl1" +
+ "&fq=host%3Alogsearch.com&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent" +
+ "&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc",
solrQuery.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
index 1428276..08541e1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
@@ -45,9 +45,9 @@ public class FieldAuditLogRequestQueryConverterTest extends AbstractRequestConve
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&" +
- "fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&" +
- "fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=10&facet.pivot=myfield%2Crepo",
+ assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+ "&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29&fq=cluster%3Acl1&facet=true" +
+ "&facet.mincount=1&facet.limit=10&facet.pivot=myfield%2Crepo",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
index 5d15166..8e96b16 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
@@ -44,9 +44,10 @@ public class ServiceLogAnyGraphRequestConverterTest extends AbstractRequestConve
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
- "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
- "&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString());
+ assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
+ "&fq=-type%3A%28hst_agent+system_message%29&fq=cluster%3Acl1&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level",
+ query.toQueryString());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
index 4378a64..981e239 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
@@ -47,8 +47,8 @@ public class ServiceLogComponentLevelRequestQueryConverterTest extends AbstractR
// THEN
assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
"&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
- "&fq=-type%3A%28hst_agent+system_message%29&fq=type%3Amycomponent&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true" +
- "&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel",
+ "&fq=-type%3A%28hst_agent+system_message%29&fq=type%3Amycomponent&fq=level%3A%28WARN+ERROR+FATAL%29&fq=cluster%3Acl1" +
+ "&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
index ce9cdcc..b72ffc5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
@@ -47,7 +47,7 @@ public class ServiceLogComponentRequestFacetQueryConverterTest extends AbstractR
// THEN
assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
"&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
- "&fq=-type%3A%28hst_agent+system_message%29&fq=type%3Amycomponent&fq=level%3A%28WARN+ERROR+FATAL%29" +
+ "&fq=-type%3A%28hst_agent+system_message%29&fq=type%3Amycomponent&fq=level%3A%28WARN+ERROR+FATAL%29&fq=cluster%3Acl1" +
"&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
index f6beb3d..69986dc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
@@ -43,9 +43,9 @@ public class ServiceLogLevelCountRequestQueryConverterTest extends AbstractReque
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
- "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
- "&fq=-type%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level",
+ assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+ "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
+ "&fq=cluster%3Acl1&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
index b941437..1987e54 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java
@@ -44,10 +44,10 @@ public class ServiceLogLevelDateRangeRequestQueryConverterTest extends AbstractR
// WHEN
SolrQuery query = underTest.convert(request);
// THEN
- assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Dlevel&facet.mincount=1&facet.limit=-1" +
- "&facet.sort=index&facet.range=%7B%21tag%3Dr1%7Dlogtime&f.logtime.facet.range.start=2016-09-13T22%3A00%3A01.000Z" +
- "&f.logtime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.logtime.facet.range.gap=%2B1HOUR&rows=0&start=0&fq=level%3A%28WARN+OR+ERROR+OR+FATAL%29",
- query.toQueryString());
+ assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Dlevel&facet.mincount=1&facet.limit=-1&facet.sort=index" +
+ "&facet.range=%7B%21tag%3Dr1%7Dlogtime&f.logtime.facet.range.start=2016-09-13T22%3A00%3A01.000Z" +
+ "&f.logtime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.logtime.facet.range.gap=%2B1HOUR&rows=0&start=0" +
+ "&fq=level%3A%28WARN+OR+ERROR+OR+FATAL%29&fq=cluster%3Acl1", query.toQueryString());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
index a140e19..167d7d8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
@@ -44,8 +44,8 @@ public class ServiceLogTreeRequestFacetQueryConverterTest extends AbstractReques
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
- "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" +
- "&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel",
+ "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
+ "&fq=level%3A%28WARN+ERROR+FATAL%29&fq=cluster%3Acl1&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
index 0151c6c..02d3f2e 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
@@ -47,7 +47,7 @@ public class ServiceLogTruncatedRequestQueryConverterTest extends AbstractReques
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&start=0&rows=10&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=logtime+desc%2Cseq_num+desc",
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=cluster%3Acl1&sort=logtime+desc%2Cseq_num+desc",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverterTest.java
index cbfab4f..66560c5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserConfigRequestQueryConverterTest.java
@@ -40,10 +40,11 @@ public class UserConfigRequestQueryConverterTest extends AbstractRequestConverte
UserConfigRequest request = new UserConfigRequest();
request.setRowType("myRowType"); // TODO: validate these 3 fields @Valid on UserConfigRequest object -> not null
request.setFilterName("myFilterName");
+ request.setClusters("cl1,cl2");
// WHEN
SolrQuery queryResult = underTest.convert(request);
// THEN
- assertEquals("?q=*%3A*&fq=rowtype%3AmyRowType&fq=filtername%3A*myFilterName*&start=0&rows=10&sort=filtername+asc",
+ assertEquals("?q=*%3A*&fq=rowtype%3AmyRowType&fq=filtername%3A*myFilterName*&fq=cluster%3A%28cl1+OR+cl2%29&start=0&rows=10&sort=filtername+asc",
queryResult.toQueryString());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edcc29f1/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
index 1608bbf..0ab57ab 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
@@ -40,6 +40,7 @@ public class UserExportRequestQueryConverterTest extends AbstractRequestConverte
UserExportRequest request = new UserExportRequest();
fillBaseLogRequestWithTestData(request);
request.setFormat("myFormat");
+ request.setClusters(null);
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
[6/6] ambari git commit: AMBARI-20022. Log Search allow installation
without portal (Miklos Gergely via oleewere)
Posted by ol...@apache.org.
AMBARI-20022. Log Search allow installation without portal (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/76ff5e0c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/76ff5e0c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/76ff5e0c
Branch: refs/heads/branch-2.5
Commit: 76ff5e0c292ded4edc1190034d04fe21ab7f119b
Parents: edcc29f
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Fri Mar 10 14:43:00 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Apr 12 14:31:27 2017 +0200
----------------------------------------------------------------------
.../LOGSEARCH/0.5.0/metainfo.xml | 2 +-
.../LOGSEARCH/0.5.0/package/scripts/params.py | 2 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 70 ++++++++++++--------
3 files changed, 46 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/76ff5e0c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index fa9548b..acf829b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -38,7 +38,7 @@
<timelineAppid>logsearch</timelineAppid>
<displayName>Log Search Server</displayName>
<category>MASTER</category>
- <cardinality>1</cardinality>
+ <cardinality>0-1</cardinality>
<versionAdvertised>false</versionAdvertised>
<commandScript>
<script>scripts/logsearch.py</script>
http://git-wip-us.apache.org/repos/asf/ambari/blob/76ff5e0c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 8dac69a..d00be4e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -391,7 +391,7 @@ logfeeder_use_ssl = logsearch_solr_ssl_enabled or metrics_collector_protocol ==
# Smoke command
#####################################
-logsearch_server_hosts = config['clusterHostInfo']['logsearch_server_hosts']
+logsearch_server_hosts = default('/configurations/clusterHostInfo/logsearch_server_hosts', None)
logsearch_server_host = ""
if logsearch_server_hosts is not None and len(logsearch_server_hosts) > 0:
logsearch_server_host = logsearch_server_hosts[0]
http://git-wip-us.apache.org/repos/asf/ambari/blob/76ff5e0c/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index fafba14..7ef5fad 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -1008,36 +1008,54 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putLogsearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env")
putLogfeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env")
- infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
-
- if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]:
- replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
- recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
+ logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER")
+ if logSearchServerHosts is None or len(logSearchServerHosts) == 0:
+ for key in services['configurations']['logsearch-env']['properties']:
+ putLogsearchEnvAttribute(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-properties']['properties']:
+ if key not in ['logsearch.collection.service.logs.numshards', 'logsearch.collection.audit.logs.numshards',
+ 'logsearch.solr.collection.service.logs', 'logsearch.solr.collection.audit.logs',
+ 'logsearch.service.logs.split.interval.mins', 'logsearch.audit.logs.split.interval.mins']:
+ putLogsearchAttribute(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']:
+ self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']:
+ self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false')
+ for key in services['configurations']['logsearch-log4j']['properties']:
+ self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false')
- recommendedMinShards = len(infraSolrHosts)
- recommendedShards = 2 * len(infraSolrHosts)
- recommendedMaxShards = 3 * len(infraSolrHosts)
+ putLogsearchProperty("logsearch.collection.service.logs.numshards", 2)
+ putLogsearchProperty("logsearch.collection.audit.logs.numshards", 2)
else:
- recommendedReplicationFactor = 2
+ infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
+ if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]:
+ replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
+ recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
+
+ recommendedMinShards = len(infraSolrHosts)
+ recommendedShards = 2 * len(infraSolrHosts)
+ recommendedMaxShards = 3 * len(infraSolrHosts)
+ else:
+ recommendedReplicationFactor = 2
+
+ recommendedMinShards = 1
+ recommendedShards = 1
+ recommendedMaxShards = 100
+
+ putLogsearchCommonEnvProperty('logsearch_use_external_solr', 'true')
+
+ # recommend number of shard
+ putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
+ putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
- recommendedMinShards = 1
- recommendedShards = 1
- recommendedMaxShards = 100
+ putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
+ putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
+ # recommend replication factor
+ putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
+ putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
- putLogsearchCommonEnvProperty('logsearch_use_external_solr', 'true')
-
- # recommend number of shard
- putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
- putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
- putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
-
- putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
- putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
- putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
- # recommend replication factor
- putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
- putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
-
kerberos_authentication_enabled = self.isSecurityEnabled(services)
if not kerberos_authentication_enabled:
putLogsearchCommonEnvProperty('logsearch_external_solr_kerberos_enabled', 'false')
[5/6] ambari git commit: AMBARI-20378. Logfeeder: add de-duplication
support (oleewere)
Posted by ol...@apache.org.
AMBARI-20378. Logfeeder: add de-duplication support (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bc97c4b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bc97c4b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bc97c4b
Branch: refs/heads/branch-2.5
Commit: 9bc97c4b998a51db4d2ed9d986fe318194f8a964
Parents: a65c5a0
Author: oleewere <ol...@gmail.com>
Authored: Fri Mar 10 14:45:13 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Apr 12 14:31:27 2017 +0200
----------------------------------------------------------------------
.../logfeeder/common/LogFeederConstants.java | 2 +
.../ambari/logfeeder/filter/FilterJSON.java | 2 +
.../apache/ambari/logfeeder/input/Input.java | 63 ++++++-
.../ambari/logfeeder/input/cache/LRUCache.java | 99 +++++++++++
.../ambari/logfeeder/mapper/MapperDate.java | 4 +
.../logfeeder/output/OutputLineFilter.java | 65 ++++++++
.../ambari/logfeeder/output/OutputManager.java | 8 +-
.../ambari/logfeeder/filter/FilterJSONTest.java | 3 +
.../logfeeder/input/cache/LRUCacheTest.java | 123 ++++++++++++++
.../ambari/logfeeder/mapper/MapperDateTest.java | 3 +
.../logfeeder/output/OutputLineFilterTest.java | 167 +++++++++++++++++++
.../logfeeder/output/OutputManagerTest.java | 3 +-
.../test-config/logfeeder/logfeeder.properties | 5 +
.../shipper-conf/input.config-zookeeper.json | 5 +-
.../configuration/logfeeder-properties.xml | 56 +++++++
15 files changed, 603 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index d1e7fba..a7559aa 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -36,4 +36,6 @@ public class LogFeederConstants {
// S3 Constants
public static final String S3_PATH_START_WITH = "s3://";
public static final String S3_PATH_SEPARATOR = "/";
+
+ public static final String IN_MEMORY_TIMESTAMP = "in_memory_timestamp";
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
index ba63c61..35f692e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
@@ -20,6 +20,7 @@ package org.apache.ambari.logfeeder.filter;
import java.util.Map;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.util.DateUtil;
@@ -48,6 +49,7 @@ public class FilterJSON extends Filter {
if (timeStampStr != null && !timeStampStr.isEmpty()) {
String logtime = DateUtil.getDate(timeStampStr);
jsonMap.put("logtime", logtime);
+ jsonMap.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, Long.parseLong(timeStampStr));
}
super.apply(jsonMap, inputMarker);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
index e13d9bd..9f54d8a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
@@ -25,12 +25,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logfeeder.input.cache.LRUCache;
import org.apache.ambari.logfeeder.common.ConfigBlock;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.output.Output;
import org.apache.ambari.logfeeder.output.OutputManager;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.log4j.Logger;
public abstract class Input extends ConfigBlock implements Runnable {
@@ -39,7 +41,18 @@ public abstract class Input extends ConfigBlock implements Runnable {
private static final boolean DEFAULT_TAIL = true;
private static final boolean DEFAULT_USE_EVENT_MD5 = false;
private static final boolean DEFAULT_GEN_EVENT_MD5 = true;
-
+ private static final boolean DEFAULT_CACHE_ENABLED = false;
+ private static final boolean DEFAULT_CACHE_DEDUP_LAST = false;
+ private static final int DEFAULT_CACHE_SIZE = 100;
+ private static final long DEFAULT_CACHE_DEDUP_INTERVAL = 1000;
+ private static final String DEFAULT_CACHE_KEY_FIELD = "log_message";
+
+ private static final String CACHE_ENABLED = "cache_enabled";
+ private static final String CACHE_KEY_FIELD = "cache_key_field";
+ private static final String CACHE_LAST_DEDUP_ENABLED = "cache_last_dedup_enabled";
+ private static final String CACHE_SIZE = "cache_size";
+ private static final String CACHE_DEDUP_INTERVAL = "cache_dedup_interval";
+
protected InputManager inputManager;
protected OutputManager outputManager;
private List<Output> outputList = new ArrayList<Output>();
@@ -54,6 +67,9 @@ public abstract class Input extends ConfigBlock implements Runnable {
private boolean useEventMD5;
private boolean genEventMD5;
+ private LRUCache cache;
+ private String cacheKeyField;
+
protected MetricData readBytesMetric = new MetricData(getReadBytesMetricName(), false);
protected String getReadBytesMetricName() {
return null;
@@ -107,6 +123,7 @@ public abstract class Input extends ConfigBlock implements Runnable {
@Override
public void init() throws Exception {
super.init();
+ initCache();
tail = getBooleanValue("tail", DEFAULT_TAIL);
useEventMD5 = getBooleanValue("use_event_md5_as_id", DEFAULT_USE_EVENT_MD5);
genEventMD5 = getBooleanValue("gen_event_md5", DEFAULT_GEN_EVENT_MD5);
@@ -114,6 +131,7 @@ public abstract class Input extends ConfigBlock implements Runnable {
if (firstFilter != null) {
firstFilter.init();
}
+
}
boolean monitor() {
@@ -217,6 +235,33 @@ public abstract class Input extends ConfigBlock implements Runnable {
isClosed = true;
}
+ private void initCache() {
+ boolean cacheEnabled = getConfigValue(CACHE_ENABLED) != null
+ ? getBooleanValue(CACHE_ENABLED, DEFAULT_CACHE_ENABLED)
+ : LogFeederUtil.getBooleanProperty("logfeeder.cache.enabled", DEFAULT_CACHE_ENABLED);
+ if (cacheEnabled) {
+ String cacheKeyField = getConfigValue(CACHE_KEY_FIELD) != null
+ ? getStringValue(CACHE_KEY_FIELD)
+ : LogFeederUtil.getStringProperty("logfeeder.cache.key.field", DEFAULT_CACHE_KEY_FIELD);
+
+ setCacheKeyField(getStringValue(cacheKeyField));
+
+ boolean cacheLastDedupEnabled = getConfigValue(CACHE_LAST_DEDUP_ENABLED) != null
+ ? getBooleanValue(CACHE_LAST_DEDUP_ENABLED, DEFAULT_CACHE_DEDUP_LAST)
+ : LogFeederUtil.getBooleanProperty("logfeeder.cache.last.dedup.enabled", DEFAULT_CACHE_DEDUP_LAST);
+
+ int cacheSize = getConfigValue(CACHE_SIZE) != null
+ ? getIntValue(CACHE_SIZE, DEFAULT_CACHE_SIZE)
+ : LogFeederUtil.getIntProperty("logfeeder.cache.size", DEFAULT_CACHE_SIZE);
+
+ long cacheDedupInterval = getConfigValue(CACHE_DEDUP_INTERVAL) != null
+ ? getLongValue(CACHE_DEDUP_INTERVAL, DEFAULT_CACHE_DEDUP_INTERVAL)
+ : Long.parseLong(LogFeederUtil.getStringProperty("logfeeder.cache.dedup.interval", String.valueOf(DEFAULT_CACHE_DEDUP_INTERVAL)));
+
+ setCache(new LRUCache(cacheSize, filePath, cacheDedupInterval, cacheLastDedupEnabled));
+ }
+ }
+
public boolean isTail() {
return tail;
}
@@ -257,6 +302,22 @@ public abstract class Input extends ConfigBlock implements Runnable {
return thread;
}
+ public LRUCache getCache() {
+ return cache;
+ }
+
+ public void setCache(LRUCache cache) {
+ this.cache = cache;
+ }
+
+ public String getCacheKeyField() {
+ return cacheKeyField;
+ }
+
+ public void setCacheKeyField(String cacheKeyField) {
+ this.cacheKeyField = cacheKeyField;
+ }
+
@Override
public String getNameForThread() {
if (filePath != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/cache/LRUCache.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/cache/LRUCache.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/cache/LRUCache.java
new file mode 100644
index 0000000..d9cfef8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/cache/LRUCache.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.input.cache;
+
+import com.google.common.collect.EvictingQueue;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * LRU cache for handle de-duplications per input files.
+ * It won't put already existing entries into the cache map if de-duplication interval not higher then a specific value
+ * or if the new value is the most recently used one (in case of lastDedupEnabled is true)
+ */
+public class LRUCache {
+ private final LinkedHashMap<String, Long> keyValueMap;
+ private final String fileName;
+ private final long dedupInterval;
+ private final boolean lastDedupEnabled;
+ private final EvictingQueue<String> mostRecentLogs;
+
+ public LRUCache(final int limit, final String fileName, final long dedupInterval, boolean lastDedupEnabled) {
+ this.fileName = fileName;
+ this.dedupInterval = dedupInterval;
+ this.lastDedupEnabled = lastDedupEnabled;
+ this.mostRecentLogs = EvictingQueue.create(1); // for now, we will just store 1 mru entry
+ keyValueMap = new LinkedHashMap<String, Long>(16, 0.75f, true) {
+ @Override
+ protected boolean removeEldestEntry(final Map.Entry<String, Long> eldest) {
+ return size() > limit;
+ }
+ };
+ }
+
+ public boolean isEntryReplaceable(String key, Long value) {
+ boolean result = true;
+ Long existingValue = keyValueMap.get(key);
+ if (existingValue == null) {
+ result = true;
+ } else if (lastDedupEnabled && mostRecentLogs.contains(key)) { // TODO: get peek element if mostRecentLogs will contain more than 1 element
+ result = false;
+ } else if (Math.abs(value - existingValue) < dedupInterval) {
+ result = false;
+ }
+ mostRecentLogs.add(key);
+ return result;
+ }
+
+ public void put(String key, Long value) {
+ if (isEntryReplaceable(key, value)) {
+ keyValueMap.put(key, value);
+ }
+ }
+
+ public Long get(String key) {
+ mostRecentLogs.add(key);
+ return keyValueMap.get(key);
+ }
+
+ public String getMRUKey() {
+ return mostRecentLogs.peek();
+ }
+
+ public int size() {
+ return keyValueMap.size();
+ }
+
+ public long getDedupInterval() {
+ return dedupInterval;
+ }
+
+ public boolean containsKey(String key) {
+ return keyValueMap.containsKey(key);
+ }
+
+ public String getFileName() {
+ return this.fileName;
+ }
+
+ public boolean isLastDedupEnabled() {
+ return lastDedupEnabled;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index eb3ae01..6a7fad7 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -24,6 +24,7 @@ import java.util.Calendar;
import java.util.Date;
import java.util.Map;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
@@ -80,6 +81,7 @@ public class MapperDate extends Mapper {
if (isEpoch) {
long ms = Long.parseLong(value.toString()) * 1000;
value = new Date(ms);
+ jsonObj.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, ((Date) value).getTime());
} else if (targetDateFormatter != null) {
if (srcDateFormatter != null) {
Date srcDate = srcDateFormatter.parse(value.toString());
@@ -97,8 +99,10 @@ public class MapperDate extends Mapper {
}
}
value = targetDateFormatter.format(srcDate);
+ jsonObj.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, srcDate.getTime());
} else {
value = targetDateFormatter.parse(value.toString());
+ jsonObj.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, ((Date) value).getTime());
}
} else {
return value;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineFilter.java
new file mode 100644
index 0000000..fcf2695
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineFilter.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.output;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.input.cache.LRUCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+
+/**
+ * Filter for outputs based on input configs, which can drop lines if the filter applies.
+ */
+public class OutputLineFilter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(OutputLineFilter.class);
+
+ /**
+ * Applies filter based on input cache (on service log only).
+ * Get the message and in-memory timestamp for log line. If both are not empty, evaluate that log line needs to be filtered out or not.
+ */
+ public Boolean apply(Map<String, Object> lineMap, Input input) {
+ boolean isLogFilteredOut = false;
+ LRUCache inputLruCache = input.getCache();
+ if (inputLruCache != null && "service".equals(input.getConfigs().get(LogFeederConstants.ROW_TYPE))) {
+ String logMessage = (String) lineMap.get(input.getCacheKeyField());
+ Long timestamp = null;
+ if (lineMap.containsKey((LogFeederConstants.IN_MEMORY_TIMESTAMP))) {
+ timestamp = (Long) lineMap.get(LogFeederConstants.IN_MEMORY_TIMESTAMP);
+ }
+ if (logMessage != null && timestamp != null) {
+ isLogFilteredOut = !inputLruCache.isEntryReplaceable(logMessage, timestamp);
+ if (!isLogFilteredOut) {
+ inputLruCache.put(logMessage, timestamp);
+ } else {
+ LOG.debug("Log line filtered out: {} (file: {}, dedupInterval: {}, lastDedupEnabled: {})",
+ logMessage, inputLruCache.getFileName(), inputLruCache.getDedupInterval(), inputLruCache.isLastDedupEnabled());
+ }
+ }
+ }
+ if (lineMap.containsKey(LogFeederConstants.IN_MEMORY_TIMESTAMP)) {
+ lineMap.remove(LogFeederConstants.IN_MEMORY_TIMESTAMP);
+ }
+ return isLogFilteredOut;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
index 86b5c57..3c80e50 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
@@ -30,8 +30,10 @@ import java.util.UUID;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.input.cache.LRUCache;
import org.apache.ambari.logfeeder.logconfig.FilterLogData;
import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logfeeder.util.DateUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.MurmurHash;
import org.apache.commons.lang3.StringUtils;
@@ -51,6 +53,8 @@ public class OutputManager {
private static long docCounter = 0;
private MetricData messageTruncateMetric = new MetricData(null, false);
+ private OutputLineFilter outputLineFilter = new OutputLineFilter();
+
public List<Output> getOutputs() {
return outputs;
}
@@ -138,8 +142,8 @@ public class OutputManager {
jsonObj.put("message_md5", "" + MurmurHash.hash64A(logMessage.getBytes(), 31174077));
}
}
-
- if (FilterLogData.INSTANCE.isAllowed(jsonObj, inputMarker)) {
+ if (FilterLogData.INSTANCE.isAllowed(jsonObj, inputMarker)
+ && !outputLineFilter.apply(jsonObj, inputMarker.input)) {
for (Output output : input.getOutputList()) {
try {
output.write(jsonObj, inputMarker);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 06d8db2..643dafc 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.OutputManager;
@@ -76,6 +77,7 @@ public class FilterJSONTest {
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Incorrect decoding: log time", dateString, jsonParams.remove("logtime"));
+ assertEquals("Incorrect decoding: in memory timestamp", d.getTime(), jsonParams.remove(LogFeederConstants.IN_MEMORY_TIMESTAMP));
assertEquals("Incorrect decoding: line number", 100l, jsonParams.remove("line_number"));
assertTrue("jsonParams are not empty!", jsonParams.isEmpty());
}
@@ -100,6 +102,7 @@ public class FilterJSONTest {
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Incorrect decoding: log time", dateString, jsonParams.remove("logtime"));
+ assertEquals("Incorrect decoding: in memory timestamp", d.getTime(), jsonParams.remove(LogFeederConstants.IN_MEMORY_TIMESTAMP));
assertEquals("Incorrect decoding: some field", "abc", jsonParams.remove("some_field"));
assertTrue("jsonParams are not empty!", jsonParams.isEmpty());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/cache/LRUCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/cache/LRUCacheTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/cache/LRUCacheTest.java
new file mode 100644
index 0000000..dd97d27
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/cache/LRUCacheTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.input.cache;
+
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class LRUCacheTest {
+
+ private LRUCache underTest;
+
+ @Before
+ public void setUp() {
+ underTest = new LRUCache(4, "/mypath", Long.parseLong("1000"), true);
+ }
+
+ @Test
+ public void testLruCachePut() {
+ // GIVEN
+ // WHEN
+ underTest.put("mymessage1", 1000L);
+ underTest.put("mymessage2", 1000L);
+ underTest.put("mymessage3", 1000L);
+ underTest.put("mymessage4", 1000L);
+ underTest.put("mymessage5", 1000L);
+ underTest.put("mymessage1", 1500L);
+ underTest.put("mymessage1", 3500L);
+ underTest.put("mymessage5", 1700L);
+ // THEN
+ assertEquals((Long) 1500L, underTest.get("mymessage1"));
+ assertEquals((Long) 1000L, underTest.get("mymessage5"));
+ assertEquals(underTest.getMRUKey(), "mymessage5");
+ assertEquals(4, underTest.size());
+ assertFalse(underTest.containsKey("mymessage2"));
+ }
+
+ @Test
+ public void testLruCacheFilterMruKeys() {
+ // GIVEN
+ // WHEN
+ underTest.put("mymessage1", 1000L);
+ underTest.put("mymessage1", 3000L);
+ underTest.put("mymessage1", 5000L);
+ underTest.put("mymessage1", 7000L);
+ // THEN
+ assertEquals((Long) 1000L, underTest.get("mymessage1"));
+ }
+
+ @Test
+ public void testLruCacheDoNotFilterMruKeysIfLastDedupDisabled() {
+ // GIVEN
+ underTest = new LRUCache(4, "/mypath", 1000, false);
+ // WHEN
+ underTest.put("mymessage1", 1000L);
+ underTest.put("mymessage1", 3000L);
+ // THEN
+ assertEquals((Long) 3000L, underTest.get("mymessage1"));
+ }
+
+ @Test
+ public void testLruCacheFilterByDedupInterval() {
+ // GIVEN
+ // WHEN
+ underTest.put("mymessage1", 1000L);
+ underTest.put("mymessage2", 1000L);
+ underTest.put("mymessage1", 1250L);
+ underTest.put("mymessage2", 1500L);
+ underTest.put("mymessage1", 1500L);
+ underTest.put("mymessage2", 2100L);
+ // THEN
+ assertEquals((Long) 1000L, underTest.get("mymessage1"));
+ assertEquals((Long) 2100L, underTest.get("mymessage2"));
+ }
+
+ @Test
+ public void testLruCacheWithDates() {
+ // GIVEN
+ DateTime firstDate = DateTime.now();
+ DateTime secondDate = firstDate.plusMillis(500);
+ // WHEN
+ underTest.put("mymessage1", firstDate.toDate().getTime());
+ underTest.put("mymessage2", firstDate.toDate().getTime());
+ underTest.put("mymessage1", secondDate.toDate().getTime());
+ // THEN
+ assertEquals((Long) firstDate.toDate().getTime(), underTest.get("mymessage1"));
+ assertEquals((Long) firstDate.toDate().getTime(), underTest.get("mymessage2"));
+ }
+
+ @Test
+ public void testLruCacheWithDatesReachDedupInterval() {
+ // GIVEN
+ DateTime firstDate = DateTime.now();
+ DateTime secondDate = firstDate.plusMillis(1500);
+ // WHEN
+ underTest.put("mymessage1", firstDate.toDate().getTime());
+ underTest.put("mymessage2", firstDate.toDate().getTime());
+ underTest.put("mymessage1", secondDate.toDate().getTime());
+ // THEN
+ assertEquals((Long) secondDate.toDate().getTime(), underTest.get("mymessage1"));
+ assertEquals((Long) firstDate.toDate().getTime(), underTest.get("mymessage2"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
index 08680f6..8beecda 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
@@ -24,6 +24,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.log4j.Logger;
import org.junit.Test;
@@ -52,6 +53,7 @@ public class MapperDateTest {
assertEquals("Value wasn't matched properly", d, mappedValue);
assertEquals("Value wasn't put into jsonObj", d, jsonObj.remove("someField"));
+ assertEquals("Value wasn't put into jsonObj", d.getTime(), jsonObj.remove(LogFeederConstants.IN_MEMORY_TIMESTAMP));
assertTrue("jsonObj is not empty", jsonObj.isEmpty());
}
@@ -73,6 +75,7 @@ public class MapperDateTest {
assertEquals("Value wasn't matched properly", d, mappedValue);
assertEquals("Value wasn't put into jsonObj", d, jsonObj.remove("someField"));
+ assertEquals("Value wasn't put into jsonObj", d.getTime(), jsonObj.remove(LogFeederConstants.IN_MEMORY_TIMESTAMP));
assertTrue("jsonObj is not empty", jsonObj.isEmpty());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java
new file mode 100644
index 0000000..1ccc319
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.output;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.input.cache.LRUCache;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class OutputLineFilterTest {
+
+ private static final String CACHE_KEY_FIELD = "log_message";
+ private static final String DEFAULT_DUMMY_MESSAGE = "myMessage";
+
+ private OutputLineFilter underTest;
+ private Input inputMock;
+
+ @Before
+ public void setUp() {
+ underTest = new OutputLineFilter();
+ inputMock = EasyMock.mock(Input.class);
+ }
+
+ @Test
+ public void testApplyWithFilterOutByDedupInterval() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 100L, false));
+ EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs());
+ EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD);
+ EasyMock.replay(inputMock);
+ // WHEN
+ boolean result = underTest.apply(generateLineMap(), inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertTrue(result);
+ }
+
+ @Test
+ public void testApplyDoNotFilterOutDataByDedupInterval() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 10L, false));
+ EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs());
+ EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD);
+ EasyMock.replay(inputMock);
+ // WHEN
+ boolean result = underTest.apply(generateLineMap(), inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertFalse(result);
+ }
+
+ @Test
+ public void testApplyWithFilterOutByDedupLast() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 10L, true));
+ EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs());
+ EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD);
+ EasyMock.replay(inputMock);
+ // WHEN
+ boolean result = underTest.apply(generateLineMap(), inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertTrue(result);
+ }
+
+ @Test
+ public void testApplyDoNotFilterOutDataByDedupLast() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(createLruCache("myMessage2", 10L, true));
+ EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs());
+ EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD);
+ EasyMock.replay(inputMock);
+ // WHEN
+ boolean result = underTest.apply(generateLineMap(), inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertFalse(result);
+ }
+
+ @Test
+ public void testApplyWithoutLruCache() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(null);
+ EasyMock.replay(inputMock);
+ // WHEN
+ boolean result = underTest.apply(generateLineMap(), inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertFalse(result);
+ }
+
+ @Test
+ public void testApplyWithoutInMemoryTimestamp() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 100L, true));
+ EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs());
+ EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD);
+ EasyMock.replay(inputMock);
+ Map<String, Object> lineMap = generateLineMap();
+ lineMap.remove(LogFeederConstants.IN_MEMORY_TIMESTAMP);
+ // WHEN
+ boolean result = underTest.apply(lineMap, inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertFalse(result);
+ }
+
+ @Test
+ public void testApplyWithoutLogMessage() {
+ // GIVEN
+ EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 100L, true));
+ EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs());
+ EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD);
+ EasyMock.replay(inputMock);
+ Map<String, Object> lineMap = generateLineMap();
+ lineMap.remove(CACHE_KEY_FIELD);
+ // WHEN
+ boolean result = underTest.apply(lineMap, inputMock);
+ // THEN
+ EasyMock.verify(inputMock);
+ assertFalse(result);
+ }
+
+ private Map<String, Object> generateLineMap() {
+ Map<String, Object> lineMap = new HashMap<>();
+ lineMap.put(CACHE_KEY_FIELD, "myMessage");
+ lineMap.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, 150L);
+ return lineMap;
+ }
+
+ private Map<String, Object> generateInputConfigs() {
+ Map<String, Object> inputConfigs = new HashMap<>();
+ inputConfigs.put(LogFeederConstants.ROW_TYPE, "service");
+ return inputConfigs;
+ }
+
+ private LRUCache createLruCache(String defaultKey, long defaultValue, boolean lastDedupEanabled) {
+ LRUCache lruCache = new LRUCache(4, "myfilepath", 100, lastDedupEanabled);
+ lruCache.put(defaultKey, defaultValue);
+ return lruCache;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
index a080fa8..0a0a195 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
@@ -103,8 +103,9 @@ public class OutputManagerTest {
expect(mockInput.isUseEventMD5()).andReturn(false);
expect(mockInput.isGenEventMD5()).andReturn(false);
expect(mockInput.getConfigs()).andReturn(Collections.<String, Object> emptyMap());
+ expect(mockInput.getCache()).andReturn(null);
expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
-
+
output1.write(jsonObj, inputMarker); expectLastCall();
output2.write(jsonObj, inputMarker); expectLastCall();
output3.write(jsonObj, inputMarker); expectLastCall();
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
index 879b786..068bc3a 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
+++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
@@ -28,3 +28,8 @@ logfeeder.log.filter.enable=true
logfeeder.solr.config.interval=5
logfeeder.solr.core.config.name=history
logfeeder.solr.zk_connect_string=localhost:9983
+logfeeder.cache.enabled=true
+logfeeder.cache.size=100
+logfeeder.cache.key.field=log_message
+logfeeder.cache.dedup.interval=1000
+logfeeder.cache.last.dedup.enabled=true
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json
index 122a9e1..d3685a4 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json
@@ -3,7 +3,10 @@
{
"type": "zookeeper",
"rowtype": "service",
- "path": "/root/test-logs/zookeeper/zookeeper-test-log.txt"
+ "path": "/root/test-logs/zookeeper/zookeeper-test-log.txt",
+ "cache_enabled" : "true",
+ "cache_size" : "10",
+ "cache_dedup_interval" : "1000"
}
],
"filter": [
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bc97c4b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
index 2acb57e..ca14c1f 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
@@ -64,4 +64,60 @@
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>logfeeder.cache.enabled</name>
+ <value>false</value>
+ <description>
+ Enable input cache for every monitored input file. The cache stores log lines, based on the data, duplications can be dropped.
+ </description>
+ <display-name>Input cache enabled</display-name>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logfeeder.cache.size</name>
+ <value>100</value>
+ <description>Size of the input caches</description>
+ <display-name>Input cache size</display-name>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logfeeder.cache.dedup.interval</name>
+ <value>1000</value>
+ <description>
+ If input cache is enabled, Log Feeder can drop any duplicated line during log processing,
+ but only if the duplicated lines/messages are in the same interval (in milliseconds) with the original message/line.
+ </description>
+ <display-name>Input cache dedup interval</display-name>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logfeeder.cache.last.dedup.enabled</name>
+ <value>false</value>
+ <description>
+ If last dedup is enabled for input cache, Log Feeder will drop every new line (message), which is the same as the last line.
+ </description>
+ <display-name>Input cache last dedup</display-name>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logfeeder.cache.key.field</name>
+ <value>log_message</value>
+ <description>
+ Key field, which will be used as keys in the Input cache. (by defalt, log_message represets the message part of processed data)<
+ /description>
+ <display-name>Input cache key field</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>