You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/11/02 18:52:08 UTC
[4/4] ambari git commit: AMBARI-18660. Log Search schema meta data
loader fix (Miklos Gergely via oleewere)
AMBARI-18660. Log Search schema meta data loader fix (Miklos Gergely via oleewere)
Change-Id: I95fad708e7cdcca23a7bdc101a3655ce9becf3b0
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/feb24a95
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/feb24a95
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/feb24a95
Branch: refs/heads/logsearch-ga
Commit: feb24a958e869171fed46f4a63977062a7b45ad9
Parents: 30beb6e
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Nov 2 19:50:51 2016 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Nov 2 19:50:51 2016 +0100
----------------------------------------------------------------------
.../ambari/logsearch/conf/SolrConfig.java | 19 +-
.../ambari/logsearch/conf/SolrPropsConfig.java | 1 -
.../logsearch/conf/SolrUserPropsConfig.java | 12 ++
.../AbstractOperationHolderConverter.java | 50 +++--
.../AbstractSearchRequestQueryConverter.java | 4 -
.../ambari/logsearch/dao/AuditSolrDao.java | 6 +-
.../logsearch/dao/ServiceLogsSolrDao.java | 3 +-
.../ambari/logsearch/dao/SolrAliasDao.java | 10 +-
.../ambari/logsearch/dao/SolrCollectionDao.java | 8 +-
.../logsearch/dao/SolrSchemaFieldDao.java | 201 ++++++++++++-------
.../ambari/logsearch/dao/UserConfigSolrDao.java | 7 +-
.../ambari/logsearch/doc/DocConstants.java | 1 +
.../logsearch/manager/AuditLogsManager.java | 7 +-
.../logsearch/manager/ServiceLogsManager.java | 8 +-
.../model/request/impl/CommonSearchRequest.java | 7 +
.../logsearch/rest/ServiceLogsResource.java | 9 +
.../apache/ambari/logsearch/util/SolrUtil.java | 93 +++------
.../webapp/scripts/views/tabs/LogFileView.js | 9 +-
.../src/main/webapp/static/schema_fields.json | 4 +-
.../converter/AuditLogRequestConverterTest.java | 2 +-
...ditServiceLoadRequestQueryConverterTest.java | 4 +-
...BaseServiceLogRequestQueryConverterTest.java | 2 +-
.../FieldAuditLogRequestQueryConverterTest.java | 2 +-
.../ServiceLogAnyGraphRequestConverterTest.java | 4 +-
...ComponentLevelRequestQueryConverterTest.java | 4 +-
...ComponentRequestFacetQueryConverterTest.java | 2 +-
...eLogLevelCountRequestQueryConverterTest.java | 2 +-
...ceLogTreeRequestFacetQueryConverterTest.java | 4 +-
...ceLogTruncatedRequestQueryConverterTest.java | 2 +-
.../UserExportRequestQueryConverterTest.java | 2 +-
.../shipper-conf/input.config-ambari.json | 20 +-
.../templates/input.config-ambari.json.j2 | 20 +-
32 files changed, 282 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
index 79df3c2..fb25a63 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
@@ -19,8 +19,6 @@
package org.apache.ambari.logsearch.conf;
import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -32,11 +30,13 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.repository.config.EnableSolrRepositories;
+import org.springframework.scheduling.annotation.EnableScheduling;
import javax.inject.Inject;
@Configuration
@EnableSolrRepositories
+@EnableScheduling
public class SolrConfig {
private static final Logger LOG = LoggerFactory.getLogger(SolrConfig.class);
@@ -80,18 +80,9 @@ public class SolrConfig {
solrUserConfigPropsConfig.getCollection()));
}
- @Bean(name = "serviceSolrFieldDao")
- public SolrSchemaFieldDao serviceSolrFieldDao() {
- return new SolrSchemaFieldDao();
- }
-
- @Bean(name = "auditSolrFieldDao")
- public SolrSchemaFieldDao auditSolrFieldDao() {
- return new SolrSchemaFieldDao();
- }
-
- @Bean(name = "userConfigSolrFieldDao")
- public SolrSchemaFieldDao userConfigSolrFieldDao() {
+ @Bean
+ @DependsOn({"serviceSolrTemplate", "auditSolrTemplate"})
+ public SolrSchemaFieldDao solrSchemaFieldDao() {
return new SolrSchemaFieldDao();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
index 78012b0..f80d6e5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
@@ -46,5 +46,4 @@ public interface SolrPropsConfig {
String getSplitInterval();
void setSplitInterval(String splitInterval);
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java
index 38366d5..c8c96c8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java
@@ -44,6 +44,9 @@ public class SolrUserPropsConfig extends SolrConnectionPropsConfig {
@Value("#{'${logsearch.logfeeder.include.default.level:FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN}'.split(',')}")
private List<String> logLevels;
+ @Value("${logsearch.schema.fields.populate.interval.mins:1}")
+ private Integer populateIntervalMins;
+
@Override
public String getCollection() {
return collection;
@@ -101,4 +104,13 @@ public class SolrUserPropsConfig extends SolrConnectionPropsConfig {
public void setLogLevels(List<String> logLevels) {
this.logLevels = logLevels;
}
+
+
+ public Integer getPopulateIntervalMins() {
+ return populateIntervalMins;
+ }
+
+ void setPopulateIntervalMins(Integer populateIntervalMins) {
+ this.populateIntervalMins = populateIntervalMins;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
index 5945d72..3c91f51 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
@@ -26,13 +26,13 @@ import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.text.StrTokenizer;
import org.springframework.data.solr.core.query.Criteria;
import org.springframework.data.solr.core.query.Query;
import org.springframework.data.solr.core.query.SimpleFilterQuery;
import org.springframework.data.solr.core.query.SimpleStringCriteria;
import javax.inject.Inject;
-import javax.inject.Named;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -43,12 +43,7 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
extends AbstractConverterAware<REQUEST_TYPE, QUERY_TYPE> {
@Inject
- @Named("serviceSolrFieldDao")
- private SolrSchemaFieldDao serviceSolrSchemaFieldDao;
-
- @Inject
- @Named("auditSolrFieldDao")
- private SolrSchemaFieldDao auditSolrSchemaFieldDao;
+ private SolrSchemaFieldDao solrSchemaFieldDao;
public List<String> splitValueAsList(String value, String separator) {
return StringUtils.isNotEmpty(value) ? Splitter.on(separator).omitEmptyStrings().splitToList(value) : null;
@@ -111,11 +106,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
new TypeToken<List<HashMap<String, String>>>(){}.getType());
for (Map<String, String> criteriaMap : criterias) {
for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
- escapeFieldValueByType(fieldEntry);
if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
- addFilterQuery(query, new Criteria(fieldEntry.getKey()).contains(escapeFieldValueByType(fieldEntry)), false);
+ addLogMessageFilter(query, fieldEntry.getValue(), false);
} else {
- addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeFieldValueByType(fieldEntry)), false);
+ addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeNonLogMessageField(fieldEntry)), false);
}
}
}
@@ -129,11 +123,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
new TypeToken<List<HashMap<String, String>>>(){}.getType());
for (Map<String, String> criteriaMap : criterias) {
for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
- escapeFieldValueByType(fieldEntry);
if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
- addFilterQuery(query, new Criteria(fieldEntry.getKey()).contains(escapeFieldValueByType(fieldEntry)), true);
+ addLogMessageFilter(query, fieldEntry.getValue(), true);
} else {
- addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeFieldValueByType(fieldEntry)), true);
+ addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeNonLogMessageField(fieldEntry)), true);
}
}
}
@@ -143,18 +136,31 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
public abstract LogType getLogType();
- private String escapeFieldValueByType(Map.Entry<String, String> fieldEntry) {
- String escapedFieldValue;
- if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
- escapedFieldValue = SolrUtil.escapeForLogMessage(fieldEntry.getValue());
- } else {
- escapedFieldValue = SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldEntry.getKey(), getSchemaFieldsTypeMapByLogType(getLogType()));
+ private void addLogMessageFilter(Query query, String value, boolean negate) {
+ StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"');
+ for (String token : tokenizer.getTokenArray()) {
+ token = token.trim();
+ if (token.contains(" ") || !token.startsWith("*") && !token.endsWith("*")) {
+ addFilterQuery(query, new Criteria(LOG_MESSAGE).is(SolrUtil.escapeQueryChars(token)), negate);
+ } else if (token.startsWith("*") && token.endsWith("*")) {
+ String plainToken = StringUtils.substring(token, 1, -1);
+ addFilterQuery(query, new Criteria(LOG_MESSAGE).contains(SolrUtil.escapeQueryChars(plainToken)), negate);
+ } else if (token.startsWith("*") && !token.endsWith("*")) {
+ String plainToken = StringUtils.substring(token, 1);
+ addFilterQuery(query, new Criteria(LOG_MESSAGE).endsWith(SolrUtil.escapeQueryChars(plainToken)), negate);
+ } else if (!token.startsWith("*") && token.endsWith("*")) {
+ String plainToken = StringUtils.substring(token, 0, -1);
+ addFilterQuery(query, new Criteria(LOG_MESSAGE).startsWith(SolrUtil.escapeQueryChars(plainToken)), negate);
+ }
}
- return escapedFieldValue;
}
- private Map<String, String> getSchemaFieldsTypeMapByLogType(LogType logType) {
- return LogType.AUDIT.equals(logType) ? auditSolrSchemaFieldDao.getSchemaFieldTypeMap() : serviceSolrSchemaFieldDao.getSchemaFieldTypeMap();
+ private String escapeNonLogMessageField(Map.Entry<String, String> fieldEntry) {
+ Map<String, String> schemaFieldNameMap = solrSchemaFieldDao.getSchemaFieldNameMap(getLogType());
+ Map<String, String> schemaFieldTypeMap = solrSchemaFieldDao.getSchemaFieldTypeMap(getLogType());
+ String fieldType = schemaFieldNameMap.get(fieldEntry.getKey());
+ String fieldTypeMetaData = schemaFieldTypeMap.get(fieldType);
+ return SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldType, fieldTypeMetaData);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
index 2604f13..d44b866 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.logsearch.converter;
-import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.model.request.impl.CommonSearchRequest;
import org.apache.commons.lang.StringUtils;
import org.springframework.data.domain.PageRequest;
@@ -27,9 +26,6 @@ import org.springframework.data.solr.core.query.Criteria;
import org.springframework.data.solr.core.query.Query;
import org.springframework.data.solr.core.query.SimpleStringCriteria;
-import javax.inject.Inject;
-import javax.inject.Named;
-
public abstract class AbstractSearchRequestQueryConverter<REQUEST_TYPE extends CommonSearchRequest, QUERY_TYPE extends Query>
extends AbstractOperationHolderConverter<REQUEST_TYPE, QUERY_TYPE> {
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
index 2fbdb46..309687d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
@@ -48,7 +48,6 @@ public class AuditSolrDao extends SolrDaoBase {
private SolrCollectionDao solrCollectionDao;
@Inject
- @Named("auditSolrFieldDao")
private SolrSchemaFieldDao solrSchemaFieldDao;
public AuditSolrDao() {
@@ -70,10 +69,9 @@ public class AuditSolrDao extends SolrDaoBase {
boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection));
solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig);
if (createAlias) {
- solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig);
- } else {
- solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig);
+ solrAliasDao.setupAlias(getSolrClient(), solrAuditLogPropsConfig);
}
+ solrSchemaFieldDao.auditCollectionSetUp();
} catch (Exception e) {
LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" +
solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e);
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
index 0e1d57c..cf61163 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
@@ -44,7 +44,6 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
private SolrTemplate serviceSolrTemplate;
@Inject
- @Named("serviceSolrFieldDao")
private SolrSchemaFieldDao solrSchemaFieldDao;
public ServiceLogsSolrDao() {
@@ -62,7 +61,7 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
try {
solrCollectionDao.checkSolrStatus(getSolrClient());
solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig);
- solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig);
+ solrSchemaFieldDao.serviceCollectionSetUp();
} catch (Exception e) {
LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl()
+ ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString()
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
index 6389446..d78ff0da 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
@@ -32,11 +32,10 @@ import javax.inject.Named;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
@Named
-public class SolrAliasDao {
+class SolrAliasDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class);
@@ -45,9 +44,9 @@ public class SolrAliasDao {
@Inject
private SolrCollectionDao solrCollectionDao;
- protected void setupAlias(final SolrSchemaFieldDao solrSchemaFieldDao, final CloudSolrClient solrClient,
- final SolrAuditLogPropsConfig solrPropsConfig) throws Exception {
- final Collection<String> collectionListIn = Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
+ void setupAlias(final CloudSolrClient solrClient, final SolrAuditLogPropsConfig solrPropsConfig) throws Exception {
+ final Collection<String> collectionListIn =
+ Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) {
LOG.info("Will not create alias " + solrPropsConfig.getAliasNameIn() + " for " +
@@ -71,7 +70,6 @@ public class SolrAliasDao {
if (count == collectionListIn.size()) {
LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " +
"Collections=" + collectionListIn);
- solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig);
break;
}
} else {
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
index c83cd99..834ba38 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
@@ -40,7 +40,7 @@ import java.util.HashSet;
import java.util.List;
@Named
-public class SolrCollectionDao {
+class SolrCollectionDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
@@ -50,7 +50,7 @@ public class SolrCollectionDao {
* This will try to get the collections from the Solr. Ping doesn't work if
* collection is not given
*/
- public boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
+ boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
int waitDurationMS = 3 * 60 * 1000;
boolean status = false;
try {
@@ -86,7 +86,7 @@ public class SolrCollectionDao {
return status;
}
- public void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
+ void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus);
if (!setupStatus) {
@@ -136,7 +136,7 @@ public class SolrCollectionDao {
}
@SuppressWarnings("unchecked")
- public List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
+ List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
IOException {
try {
CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index 35bc2dc..d05e650 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -19,123 +19,178 @@
package org.apache.ambari.logsearch.dao;
import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.LukeRequest;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.LukeResponse;
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.NamedList;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
-import javax.inject.Inject;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import javax.inject.Named;
+
public class SolrSchemaFieldDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
- private static final int SETUP_RETRY_SECOND = 30;
- private static final int SETUP_UPDATE_SECOND = 1 * 60; // 1 min
+ private static final int RETRY_SECOND = 30;
- private boolean populateFieldsThreadActive = false;
-
- private Map<String, String> schemaFieldNameMap = new HashMap<>();
- private Map<String, String> schemaFieldTypeMap = new HashMap<>();
+ @Inject
+ @Named("serviceSolrTemplate")
+ private SolrTemplate serviceSolrTemplate;
@Inject
- private SolrUserPropsConfig solrUserPropsConfig;
+ @Named("auditSolrTemplate")
+ private SolrTemplate auditSolrTemplate;
+
+ @Inject
+ private SolrUserPropsConfig solrUserConfigPropsConfig;
+
+ private CloudSolrClient serviceSolrClient;
+ private CloudSolrClient auditSolrClient;
+
+ private int retryCount;
+ private int skipCount;
- public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) {
- if (!populateFieldsThreadActive) {
- populateFieldsThreadActive = true;
- LOG.info("Creating thread to populated fields for collection=" + solrPropsConfig.getCollection());
- Thread fieldPopulationThread = new Thread("populated_fields_" + solrPropsConfig.getCollection()) {
- @Override
- public void run() {
- LOG.info("Started thread to get fields for collection=" + solrPropsConfig.getCollection());
- int retryCount = 0;
- while (true) {
- try {
- Thread.sleep(SETUP_RETRY_SECOND * 1000);
- retryCount++;
- boolean _result = _populateSchemaFields(solrClient, solrPropsConfig);
- if (_result) {
- LOG.info("Populate fields for collection " + solrPropsConfig.getCollection() + " is success, Update it after " +
- SETUP_UPDATE_SECOND + " sec");
- Thread.sleep(SETUP_UPDATE_SECOND * 1000);
- }
- } catch (InterruptedException sleepInterrupted) {
- LOG.info("Sleep interrupted while populating fields for collection " + solrPropsConfig.getCollection());
- break;
- } catch (Exception ex) {
- LOG.error("Error while populating fields for collection " + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
- }
- }
- populateFieldsThreadActive = false;
- LOG.info("Exiting thread for populating fields. collection=" + solrPropsConfig.getCollection());
- }
- };
- fieldPopulationThread.setDaemon(true);
- fieldPopulationThread.start();
+ private boolean serviceCollectionSetUp = false;
+ private boolean auditCollectionSetUp = false;
+
+ private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
+ private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
+ private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
+ private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
+
+ @PostConstruct
+ public void init() {
+ this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient();
+ this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient();
+ }
+
+ void serviceCollectionSetUp() {
+ this.serviceCollectionSetUp = true;
+ }
+
+ void auditCollectionSetUp() {
+ this.auditCollectionSetUp = true;
+ }
+
+ @Scheduled(fixedDelay = RETRY_SECOND * 1000)
+ public void populateAllSchemaFields() {
+ if (skipCount > 0) {
+ skipCount--;
+ return;
+ }
+
+ if (serviceCollectionSetUp) {
+ populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap);
+ }
+ if (auditCollectionSetUp) {
+ populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap);
}
}
- /**
- * Called from the thread. Don't call this directly
- */
- private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
- String historyCollection = solrUserPropsConfig.getCollection();
- if (solrClient != null && !solrPropsConfig.getCollection().equals(historyCollection)) {
- LukeResponse lukeResponse = null;
+ private void populateSchemaFields(CloudSolrClient solrClient, Map<String, String> schemaFieldNameMap,
+ Map<String, String> schemaFieldTypeMap) {
+ if (solrClient != null) {
+ LOG.info("Started thread to get fields for collection=" + solrClient.getDefaultCollection());
+ List<LukeResponse> lukeResponses = null;
SchemaResponse schemaResponse = null;
try {
- LukeRequest lukeRequest = new LukeRequest();
- lukeRequest.setNumTerms(0);
- lukeResponse = lukeRequest.process(solrClient);
-
+ lukeResponses = getLukeResponsesForCores(solrClient);
+
SolrRequest<SchemaResponse> schemaRequest = new SchemaRequest();
schemaRequest.setMethod(SolrRequest.METHOD.GET);
schemaRequest.setPath("/schema");
schemaResponse = schemaRequest.process(solrClient);
- LOG.debug("populateSchemaFields() collection=" + solrPropsConfig.getCollection() + ", luke=" + lukeResponse +
+ LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses +
", schema= " + schemaResponse);
} catch (SolrException | SolrServerException | IOException e) {
- LOG.error("Error occured while popuplating field. collection=" + solrPropsConfig.getCollection(), e);
+ LOG.error("Error occured while popuplating field. collection=" + solrClient.getDefaultCollection(), e);
}
- if (lukeResponse != null && schemaResponse != null) {
- extractSchemaFieldsName(lukeResponse, schemaResponse);
- return true;
+ if (schemaResponse != null) {
+ extractSchemaFieldsName(lukeResponses, schemaResponse, schemaFieldNameMap, schemaFieldTypeMap);
+ LOG.info("Populate fields for collection " + solrClient.getDefaultCollection()+ " was successful, next update it after " +
+ solrUserConfigPropsConfig.getPopulateIntervalMins() + " minutes");
+ retryCount = 0;
+ skipCount = (solrUserConfigPropsConfig.getPopulateIntervalMins() * 60) / RETRY_SECOND - 1;
+ }
+ else {
+ retryCount++;
+ LOG.error("Error while populating fields for collection " + solrClient.getDefaultCollection() + ", retryCount=" + retryCount);
}
}
- return false;
+ }
+
+ private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2";
+
+ @SuppressWarnings("unchecked")
+ private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
+ ZkStateReader zkStateReader = solrClient.getZkStateReader();
+ Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection());
+
+ List<LukeResponse> lukeResponses = new ArrayList<>();
+ for (Slice slice : activeSlices) {
+ for (Replica replica : slice.getReplicas()) {
+ try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
+ HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
+ HttpResponse response = httpClient.execute(request);
+ NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent());
+ LukeResponse lukeResponse = new LukeResponse();
+ lukeResponse.setResponse(lukeData);
+ lukeResponses.add(lukeResponse);
+ } catch (IOException e) {
+ LOG.error("Exception during getting luke responses", e);
+ }
+ }
+ }
+ return lukeResponses;
}
- private void extractSchemaFieldsName(LukeResponse lukeResponse, SchemaResponse schemaResponse) {
+ private void extractSchemaFieldsName(List<LukeResponse> lukeResponses, SchemaResponse schemaResponse,
+ Map<String, String> schemaFieldNameMap, Map<String, String> schemaFieldTypeMap) {
try {
HashMap<String, String> _schemaFieldNameMap = new HashMap<>();
HashMap<String, String> _schemaFieldTypeMap = new HashMap<>();
- for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) {
- String name = e.getKey();
- String type = e.getValue().getType();
- if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") &&
- !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) {
- _schemaFieldNameMap.put(name, type);
+ for (LukeResponse lukeResponse : lukeResponses) {
+ for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) {
+ String name = e.getKey();
+ String type = e.getValue().getType();
+ if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") &&
+ !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) {
+ _schemaFieldNameMap.put(name, type);
+ }
}
}
@@ -162,19 +217,21 @@ public class SolrSchemaFieldDao {
}
synchronized (this) {
- schemaFieldNameMap = _schemaFieldNameMap;
- schemaFieldTypeMap = _schemaFieldTypeMap;
+ schemaFieldNameMap.clear();
+ schemaFieldNameMap.putAll(_schemaFieldNameMap);
+ schemaFieldTypeMap.clear();
+ schemaFieldTypeMap.putAll(_schemaFieldTypeMap);
}
} catch (Exception e) {
LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM);
}
}
- public synchronized Map<String, String> getSchemaFieldNameMap() {
- return schemaFieldNameMap;
+ public Map<String, String> getSchemaFieldNameMap(LogType logType) {
+ return LogType.AUDIT == logType ? auditSchemaFieldNameMap : serviceSchemaFieldNameMap;
}
- public synchronized Map<String, String> getSchemaFieldTypeMap() {
- return schemaFieldTypeMap;
+ public Map<String, String> getSchemaFieldTypeMap(LogType logType) {
+ return LogType.AUDIT == logType ? auditSchemaFieldTypeMap : serviceSchemaFieldTypeMap;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
index 418a405..106bedc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
@@ -69,10 +69,6 @@ public class UserConfigSolrDao extends SolrDaoBase {
private SolrCollectionDao solrCollectionDao;
@Inject
- @Named("userConfigSolrFieldDao")
- private SolrSchemaFieldDao solrSchemaFieldDao;
-
- @Inject
@Named("userConfigSolrTemplate")
private SolrTemplate userConfigSolrTemplate;
@@ -94,7 +90,6 @@ public class UserConfigSolrDao extends SolrDaoBase {
try {
solrCollectionDao.checkSolrStatus(getSolrClient());
solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
- solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig);
intializeLogFeederFilter();
} catch (Exception e) {
@@ -185,6 +180,6 @@ public class UserConfigSolrDao extends SolrDaoBase {
@Override
public SolrSchemaFieldDao getSolrSchemaFieldDao() {
- return solrSchemaFieldDao;
+ throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index e6f5103..3639b88 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -90,6 +90,7 @@ public class DocConstants {
public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "Get service logs schema fields";
public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch";
public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data";
+ public static final String REQUEST_CANCEL = "Cancel an ongoing solr request";
}
public class PublicOperationDescriptions {
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
index aded1d4..3ee1d17 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
@@ -36,8 +36,11 @@ import java.util.Map;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
+
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.graph.GraphDataGenerator;
import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest;
@@ -79,6 +82,8 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
private ConversionService conversionService;
@Inject
private Configuration freemarkerConfiguration;
+ @Inject
+ private SolrSchemaFieldDao solrSchemaFieldDao;
public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) {
return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs");
@@ -118,7 +123,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
public String getAuditLogsSchemaFieldsName() {
- return convertObjToString(auditSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap());
+ return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.AUDIT));
}
public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
index 44d0c00..8986544 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
@@ -28,7 +28,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Scanner;
import javax.inject.Inject;
import javax.inject.Named;
@@ -42,8 +41,10 @@ import freemarker.template.TemplateException;
import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
+import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.graph.GraphDataGenerator;
import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest;
import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest;
@@ -71,7 +72,6 @@ import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData;
import org.apache.ambari.logsearch.solr.model.SolrHostLogData;
import org.apache.ambari.logsearch.solr.model.SolrServiceLogData;
import org.apache.ambari.logsearch.util.DownloadUtil;
-import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.ambari.logsearch.util.DateUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
import org.apache.ambari.logsearch.util.SolrUtil;
@@ -116,6 +116,8 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
private ConversionService conversionService;
@Inject
private Configuration freemarkerConfiguration;
+ @Inject
+ private SolrSchemaFieldDao solrSchemaFieldDao;
public ServiceLogResponse searchLogs(ServiceLogRequest request) {
String event = "/service/logs";
@@ -418,7 +420,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
public String getServiceLogsSchemaFieldsName() {
- return convertObjToString(serviceLogsSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap());
+ return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.SERVICE));
}
public BarGraphDataListResponse getAnyGraphCountData(ServiceAnyGraphRequest request) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
index 7001cf3..2617c51 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
@@ -21,6 +21,8 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.CommonSearchParamDefinition;
import org.apache.ambari.logsearch.model.request.SearchRequest;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import javax.ws.rs.QueryParam;
@@ -116,4 +118,9 @@ public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefi
public void setEndTime(String endTime) {
this.endTime = endTime;
}
+
+ @Override
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
index 492a5b2..3e1e120 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
@@ -184,4 +184,13 @@ public class ServiceLogsResource {
public String getHadoopServiceConfigJSON() {
return serviceLogsManager.getHadoopServiceConfigJSON();
}
+
+ @GET
+ @Path("/request/cancel")
+ @Produces({"application/json"})
+ @ApiOperation(REQUEST_CANCEL)
+ public String cancelRequest() {
+ // TODO: create function that cancels an ongoing solr request
+ return "{\"endpoint status\": \"not supported yet\"}";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
index 7a856d2..a3824de 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
@@ -35,7 +35,7 @@ import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public class SolrUtil {
private SolrUtil() {
@@ -74,50 +74,14 @@ public class SolrUtil {
if (search == null) {
return null;
}
- String newString = search.trim();
- String newSearch = escapeQueryChars(newString);
- boolean isSingleWord = true;
- for (int i = 0; i < search.length(); i++) {
- if (Character.isWhitespace(search.charAt(i))) {
- isSingleWord = false;
- }
- }
- if (!isSingleWord) {
+ String newSearch = escapeQueryChars(search.trim());
+ if (StringUtils.containsWhitespace(newSearch)) {
newSearch = "\"" + newSearch + "\"";
}
return newSearch;
}
- private static String escapeForKeyTokenizer(String search) {
- if (search.startsWith("*") && search.endsWith("*") && StringUtils.isNotBlank(search)) {
- // Remove the * from both the sides
- if (search.length() > 1) {
- search = search.substring(1, search.length() - 1);
- } else {
- //search string have only *
- search="";
- }
- }
- search = escapeQueryChars(search);
-
- return "*" + search + "*";
- }
-
- /**
- * This is a special case scenario to handle log_message for wild card
- * scenarios
- */
- public static String escapeForLogMessage(String search) {
- if (search.startsWith("*") && search.endsWith("*")) {
- search = escapeForKeyTokenizer(search);
- } else {
- // Use whitespace index
- search = escapeForStandardTokenizer(search);
- }
- return search;
- }
-
private static String makeSolrSearchStringWithoutAsterisk(String search) {
String newString = search.trim();
String newSearch = newString.replaceAll("(?=[]\\[+&|!(){}^\"~=/$@%?:.\\\\])", "\\\\");
@@ -152,48 +116,45 @@ public class SolrUtil {
}
- private static boolean isSolrFieldNumber(String fieldType, Map<String, String> schemaFieldsMap) {
- if (StringUtils.isBlank(fieldType)) {
+ private static boolean isSolrFieldNumber(Map<String, Object> fieldTypeInfoMap) {
+ if (MapUtils.isEmpty(fieldTypeInfoMap)) {
return false;
- } else {
- HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType, schemaFieldsMap);
- if (MapUtils.isEmpty(typeInfoMap)) {
- return false;
- }
- String fieldTypeClassName = (String) typeInfoMap.get("class");
- return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) ||
- fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) ||
- fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) ||
- fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName());
}
+ String fieldTypeClassName = (String) fieldTypeInfoMap.get("class");
+ return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) ||
+ fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) ||
+ fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) ||
+ fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName());
}
- public static String putWildCardByType(String str, String key, Map<String, String> schemaFieldsMap) {
- String fieldType = schemaFieldsMap.get(key);
+ public static String putWildCardByType(String str, String fieldType, String fieldTypeMetaData) {
+ Map<String, Object> fieldTypeInfoMap = getFieldTypeInfoMap(fieldTypeMetaData);
if (StringUtils.isNotBlank(fieldType)) {
- if (isSolrFieldNumber(fieldType, schemaFieldsMap)) {
- String value = putEscapeCharacterForNumber(str, fieldType, schemaFieldsMap);
+ if (isSolrFieldNumber(fieldTypeInfoMap)) {
+ String value = putEscapeCharacterForNumber(str, fieldTypeInfoMap);
if (StringUtils.isNotBlank(value)) {
return value;
} else {
return null;
}
- } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class, schemaFieldsMap)) {
+ } else if (checkTokenizer(StandardTokenizerFactory.class, fieldTypeInfoMap)) {
return escapeForStandardTokenizer(str);
- } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class, schemaFieldsMap)|| "string".equalsIgnoreCase(fieldType)) {
+ } else if (checkTokenizer(KeywordTokenizerFactory.class, fieldTypeInfoMap) || "string".equalsIgnoreCase(fieldType)) {
return makeSolrSearchStringWithoutAsterisk(str);
- } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class, schemaFieldsMap)) {
+ } else if (checkTokenizer(PathHierarchyTokenizerFactory.class, fieldTypeInfoMap)) {
return str;
+ } else {
+ return escapeQueryChars(str);
}
}
return str;
}
- private static String putEscapeCharacterForNumber(String str,String fieldType, Map<String, String> schemaFieldsMap) {
+ private static String putEscapeCharacterForNumber(String str, Map<String, Object> fieldTypeInfoMap) {
if (StringUtils.isNotEmpty(str)) {
str = str.replace("*", "");
}
- String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType, schemaFieldsMap);
+ String escapeCharSting = parseInputValueAsPerFieldType(str, fieldTypeInfoMap);
if (escapeCharSting == null || escapeCharSting.isEmpty()) {
return null;
}
@@ -201,9 +162,8 @@ public class SolrUtil {
return escapeCharSting;
}
- private static String parseInputValueAsPerFieldType(String str,String fieldType, Map<String, String> schemaFieldsMap) {
+ private static String parseInputValueAsPerFieldType(String str, Map<String, Object> fieldTypeInfoMap) {
try {
- HashMap<String, Object> fieldTypeInfoMap = SolrUtil.getFieldTypeInfoMap(fieldType, schemaFieldsMap);
String className = (String) fieldTypeInfoMap.get("class");
if (className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())) {
return "" + Double.parseDouble(str);
@@ -219,8 +179,7 @@ public class SolrUtil {
}
}
- private static HashMap<String, Object> getFieldTypeInfoMap(String fieldType, Map<String, String> schemaFieldsTypeMap) {
- String fieldTypeMetaData = schemaFieldsTypeMap.get(fieldType);
+ private static Map<String, Object> getFieldTypeInfoMap(String fieldTypeMetaData) {
HashMap<String, Object> fieldTypeMap = JSONUtil.jsonToMapObject(fieldTypeMetaData);
if (fieldTypeMap == null) {
return new HashMap<>();
@@ -269,10 +228,8 @@ public class SolrUtil {
}
@SuppressWarnings("unchecked")
- private static boolean checkTokenizer(String fieldType, Class<? extends TokenizerFactory> tokenizerFactoryClass,
- Map<String, String> schemaFieldsMap) {
- HashMap<String, Object> fieldTypeMap = SolrUtil.getFieldTypeInfoMap(fieldType ,schemaFieldsMap);
- HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap.get("analyzer");
+ private static boolean checkTokenizer(Class<? extends TokenizerFactory> tokenizerFactoryClass, Map<String, Object> fieldTypeInfoMap) {
+ HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeInfoMap.get("analyzer");
HashMap<String, Object> tokenizerMap = (HashMap<String, Object>)MapUtils.getObject(analyzer, "tokenizer");
if (tokenizerMap != null) {
String tokenizerClass = (String) tokenizerMap.get("class");
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
index a4e3a1a..57d4acd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
@@ -596,12 +596,13 @@ define(['require',
this.ui.find.trigger("keyup");
this.ui.find.focus();
}else if(type === "IA" || type === "EA"){
- this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns",
- key:Globals.serviceLogsColumns["log_message"],value:"*"+this.selectionText+"*"});
- }
+ this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns",
+ key:Globals.defaultServiceLogMappings["log_message"],value:"*"+this.selectionText+"*"});
+ }
else {
//this.vent.trigger("add:include:exclude",{type:type,value:this.selectionText});
- this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns", key: Globals.serviceLogsColumns["log_message"], value: this.selectionText });
+ this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns",
+ key: Globals.defaultServiceLogMappings["log_message"], value: this.selectionText });
}
this.ui.contextMenu.hide();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json
index 55f1d3c..d145141 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json
@@ -46,7 +46,7 @@
"logType": "Log Type",
"logfile_line_number": "Logfile Line Number",
"logger_name": "Logger Name",
- "logtime": "logtime",
+ "logtime": "Log Time",
"path": "Path",
"perm": "Perm",
"policy": "Policy",
@@ -76,11 +76,13 @@
"ws_repo_id": "Repo Id",
"ws_repo_version": "Repo Version",
"ws_repositories": "Repositories",
+ "ws_request_id": "Request Id",
"ws_result_status": "Result Status",
"ws_roles": "Roles",
"ws_stack_version": "Stack Version",
"ws_stack": "Stack",
"ws_status": "Status",
+ "ws_task_id": "Task Id",
"ws_version_note": "Version Note",
"ws_version_number": "Version Number"
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
index ec6aad7..1aae82a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
@@ -46,7 +46,7 @@ public class AuditLogRequestConverterTest extends AbstractRequestConverterTest {
SolrQuery queryResult = new DefaultQueryParser().doConstructSolrQuery(simpleQuery);
// THEN
assertEquals("?q=*%3A*&start=0&rows=25&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&sort=evtTime+desc%2Cseq_num+desc",
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=evtTime+desc%2Cseq_num+desc",
queryResult.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
index 71e91e1..2b629e5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java
@@ -43,8 +43,8 @@ public class AuditServiceLoadRequestQueryConverterTest extends AbstractRequestCo
// WHEN
SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*&" +
- "fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
+ assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage&" +
+ "fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
"&facet=true&facet.mincount=1&facet.limit=10&facet.field=repo", solrQuery.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
index 3775a2d..89f8eba 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java
@@ -53,7 +53,7 @@ public class BaseServiceLogRequestQueryConverterTest extends AbstractRequestConv
SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery);
// THEN
assertEquals("?q=*%3A*&start=0&rows=25&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=host%3Alogsearch.com" +
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=host%3Alogsearch.com" +
"&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" +
"&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc",
solrQuery.toQueryString());
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
index 1864af3..1428276 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java
@@ -46,7 +46,7 @@ public class FieldAuditLogRequestQueryConverterTest extends AbstractRequestConve
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&" +
- "fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29&" +
+ "fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&" +
"fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=10&facet.pivot=myfield%2Crepo",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
index 63dc733..5d15166 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java
@@ -44,8 +44,8 @@ public class ServiceLogAnyGraphRequestConverterTest extends AbstractRequestConve
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" +
- "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
+ assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+ "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
"&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
index 2a7cc16..b1ae332 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java
@@ -45,8 +45,8 @@ public class ServiceLogComponentLevelRequestQueryConverterTest extends AbstractR
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" +
- "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
+ assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+ "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
"&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
index a5ea683..3c7f242 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java
@@ -46,7 +46,7 @@ public class ServiceLogComponentRequestFacetQueryConverterTest extends AbstractR
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
- "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" +
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
"&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1" +
"&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel",
query.toQueryString());
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
index b79ff91..f6beb3d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java
@@ -44,7 +44,7 @@ public class ServiceLogLevelCountRequestQueryConverterTest extends AbstractReque
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
- "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" +
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
"&fq=-type%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
index 634ac19..107a99d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java
@@ -44,8 +44,8 @@ public class ServiceLogTreeRequestFacetQueryConverterTest extends AbstractReques
// WHEN
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
- assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" +
- "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" +
+ assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+ "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" +
"&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
index 41761bc..0151c6c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java
@@ -47,7 +47,7 @@ public class ServiceLogTruncatedRequestQueryConverterTest extends AbstractReques
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&start=0&rows=10&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&sort=logtime+desc%2Cseq_num+desc",
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=logtime+desc%2Cseq_num+desc",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
index 6644ca1..1608bbf 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java
@@ -44,7 +44,7 @@ public class UserExportRequestQueryConverterTest extends AbstractRequestConverte
SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
// THEN
assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
- "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29" +
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29" +
"&fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.pivot=reqUser%2Crepo&facet.pivot=resource%2Crepo",
query.toQueryString());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json
index 576bfa8..2dfe6d3 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json
@@ -282,16 +282,6 @@
"new_fieldname":"cliType"
}
},
- "RequestId":{
- "map_fieldname":{
- "new_fieldname":"request_id"
- }
- },
- "TaskId":{
- "map_fieldname":{
- "new_fieldname":"task_id"
- }
- },
"Operation":{
"map_fieldname":{
"new_fieldname":"action"
@@ -374,6 +364,11 @@
"new_fieldname":"ws_repositories"
}
},
+ "RequestId":{
+ "map_fieldname":{
+ "new_fieldname":"ws_request_id"
+ }
+ },
"Roles":{
"map_fieldname":{
"new_fieldname":"ws_roles"
@@ -389,6 +384,11 @@
"new_fieldname":"ws_stack_version"
}
},
+ "TaskId":{
+ "map_fieldname":{
+ "new_fieldname":"ws_task_id"
+ }
+ },
"VersionNote":{
"map_fieldvalue":{
"pre_value":"null",
http://git-wip-us.apache.org/repos/asf/ambari/blob/feb24a95/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
index ad4adb2..f03f9ea 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
@@ -299,16 +299,6 @@
"new_fieldname":"cliType"
}
},
- "RequestId":{
- "map_fieldname":{
- "new_fieldname":"request_id"
- }
- },
- "TaskId":{
- "map_fieldname":{
- "new_fieldname":"task_id"
- }
- },
"Operation":{
"map_fieldname":{
"new_fieldname":"action"
@@ -391,6 +381,11 @@
"new_fieldname":"ws_repositories"
}
},
+ "RequestId":{
+ "map_fieldname":{
+ "new_fieldname":"ws_request_id"
+ }
+ },
"Roles":{
"map_fieldname":{
"new_fieldname":"ws_roles"
@@ -406,6 +401,11 @@
"new_fieldname":"ws_stack_version"
}
},
+ "TaskId":{
+ "map_fieldname":{
+ "new_fieldname":"ws_task_id"
+ }
+ },
"VersionNote":{
"map_fieldvalue":{
"pre_value":"null",