You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/09/28 09:42:18 UTC
[51/52] [abbrv] ambari git commit: AMBARI-18310. Logsearch - Refactor
solr query layer (oleewere)
AMBARI-18310. Logsearch - Refactor solr query layer (oleewere)
Change-Id: I0e99217fea1de03135b86e007fcb2a3ff6c2e257
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/44644cd6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/44644cd6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/44644cd6
Branch: refs/heads/logsearch-ga
Commit: 44644cd619f6ab9efb8ae2e579a5a9125efcc964
Parents: 47bf18d
Author: oleewere <ol...@gmail.com>
Authored: Mon Sep 12 21:35:08 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Sep 28 11:32:24 2016 +0200
----------------------------------------------------------------------
.../ambari-logsearch-portal/pom.xml | 10 +
.../logsearch/common/ExternalServerClient.java | 4 +-
.../logsearch/common/LogSearchConstants.java | 51 +-
.../apache/ambari/logsearch/common/LogType.java | 34 +
.../logsearch/conf/ApplicationConfig.java | 13 +
.../ambari/logsearch/conf/SolrConfig.java | 11 +
.../ambari/logsearch/dao/AuditSolrDao.java | 23 +-
.../logsearch/dao/ServiceLogsSolrDao.java | 21 +-
.../ambari/logsearch/dao/SolrAliasDao.java | 8 +-
.../ambari/logsearch/dao/SolrCollectionDao.java | 4 +-
.../ambari/logsearch/dao/SolrDaoBase.java | 99 +-
.../logsearch/dao/SolrSchemaFieldDao.java | 24 +-
.../ambari/logsearch/dao/UserConfigSolrDao.java | 23 +-
.../ambari/logsearch/doc/DocConstants.java | 18 +-
.../logsearch/graph/GraphDataGenerator.java | 625 +++---
.../logsearch/graph/GraphDataGeneratorBase.java | 169 --
.../logsearch/manager/AuditLogsManager.java | 478 +----
.../ambari/logsearch/manager/ManagerBase.java | 137 +-
.../ambari/logsearch/manager/PublicManager.java | 4 +-
.../logsearch/manager/ServiceLogsManager.java | 1771 +++---------------
.../logsearch/manager/SessionManager.java | 5 +-
.../logsearch/manager/UserConfigManager.java | 70 +-
.../model/request/LogParamDefinition.java | 7 +
.../request/LogTruncatedParamDefinition.java | 4 +-
.../model/request/QueryParamDefinition.java | 32 -
.../request/ServiceLogParamDefinition.java | 13 -
.../model/request/TopParamDefinition.java | 31 +
.../model/request/impl/AnyGraphRequest.java | 109 --
.../request/impl/AuditBarGraphRequest.java | 2 +-
.../request/impl/AuditComponentRequest.java | 25 +
.../model/request/impl/AuditLogRequest.java | 2 +-
.../request/impl/AuditServiceLoadRequest.java | 25 +
.../model/request/impl/BaseAuditLogRequest.java | 53 -
.../model/request/impl/BaseLogRequest.java | 43 +-
.../request/impl/BaseServiceLogRequest.java | 54 +-
.../request/impl/FieldAuditLogRequest.java | 18 +-
.../model/request/impl/QueryRequest.java | 40 -
.../impl/ServiceExtremeDatesRequest.java | 41 -
.../impl/ServiceLogAggregatedInfoRequest.java | 25 +
.../impl/ServiceLogComponentHostRequest.java | 39 +
.../impl/ServiceLogComponentLevelRequest.java | 25 +
.../impl/ServiceLogHostComponentRequest.java | 39 +
.../impl/ServiceLogLevelCountRequest.java | 25 +
.../impl/ServiceLogTruncatedRequest.java | 6 +-
.../model/request/impl/SimpleQueryRequest.java | 42 -
.../logsearch/model/response/TemplateData.java | 36 +
.../ambari/logsearch/query/QueryGeneration.java | 340 ----
.../logsearch/query/QueryGenerationBase.java | 282 ---
.../AbstractAuditLogRequestQueryConverter.java | 47 +
.../AbstractCommonAuditLogRequestConverter.java | 44 -
.../AbstractCommonSearchRequestConverter.java | 55 -
...bstractCommonServiceLogRequestConverter.java | 51 -
.../query/converter/AbstractConverterAware.java | 4 +-
.../AbstractDateRangeFacetQueryConverter.java | 55 +
.../AbstractLogRequestFacetQueryConverter.java | 61 +
.../AbstractLogRequestQueryConverter.java | 44 +
.../AbstractSearchRequestQueryConverter.java | 173 ++
.../converter/AnyGraphRequestConverter.java | 39 -
.../AuditBarGraphRequestConverter.java | 34 -
.../AuditBarGraphRequestQueryConverter.java | 40 +
.../AuditComponentsRequestQueryConverter.java | 62 +
.../converter/AuditLogRequestConverter.java | 34 -
.../AuditLogRequestQueryConverter.java | 44 +
.../AuditServiceLoadRequestQueryConverter.java | 47 +
.../converter/BaseAuditLogRequestConverter.java | 33 -
.../BaseServiceLogRequestConverter.java | 33 -
.../BaseServiceLogRequestQueryConverter.java | 82 +
.../FieldAuditLogRequestConverter.java | 34 -
.../FieldAuditLogRequestQueryConverter.java | 48 +
.../FieldBarGraphRequestConverter.java | 35 -
.../ServiceAnyGraphRequestConverter.java | 39 -
.../ServiceExtremeDatesRequestConverter.java | 34 -
.../converter/ServiceGraphRequestConverter.java | 34 -
...ServiceLogAnyGraphRequestQueryConverter.java | 46 +
...eLogComponentLevelRequestQueryConverter.java | 61 +
...eLogComponentRequestFacetQueryConverter.java | 62 +
.../ServiceLogExportRequestConverter.java | 35 -
...rviceLogLevelCountRequestQueryConverter.java | 46 +
...eLogLevelDateRangeRequestQueryConverter.java | 55 +
.../converter/ServiceLogRequestConverter.java | 39 -
...erviceLogTreeRequestFacetQueryConverter.java | 63 +
.../ServiceLogTruncatedRequestConverter.java | 36 -
...erviceLogTruncatedRequestQueryConverter.java | 95 +
.../converter/SimpleQueryRequestConverter.java | 35 -
.../StringFieldFacetQueryConverter.java | 44 +
.../converter/UserExportRequestConverter.java | 36 -
.../UserExportRequestQueryConverter.java | 49 +
.../query/model/AnyGraphSearchCriteria.java | 77 -
.../model/AuditBarGraphSearchCriteria.java | 33 -
.../query/model/AuditLogSearchCriteria.java | 33 -
.../model/CommonServiceLogSearchCriteria.java | 88 -
.../model/FieldAuditBarGraphSearchCriteria.java | 32 -
.../model/FieldAuditLogSearchCriteria.java | 32 -
.../model/ServiceAnyGraphSearchCriteria.java | 60 -
.../model/ServiceExtremeDatesCriteria.java | 32 -
.../query/model/ServiceGraphSearchCriteria.java | 32 -
.../model/ServiceLogExportSearchCriteria.java | 40 -
.../query/model/ServiceLogSearchCriteria.java | 65 -
.../ServiceLogTruncatedSearchCriteria.java | 48 -
.../query/model/UserExportSearchCriteria.java | 31 -
.../logsearch/rest/AuditLogsResource.java | 79 +-
.../ambari/logsearch/rest/PublicResource.java | 4 +-
.../logsearch/rest/ServiceLogsResource.java | 88 +-
.../logsearch/rest/UserConfigResource.java | 4 +-
.../ambari/logsearch/solr/SolrConstants.java | 62 +
.../apache/ambari/logsearch/util/BizUtil.java | 261 ---
.../apache/ambari/logsearch/util/DateUtil.java | 9 +
.../ambari/logsearch/util/DownloadUtil.java | 176 ++
.../apache/ambari/logsearch/util/FileUtil.java | 84 -
.../apache/ambari/logsearch/util/SolrUtil.java | 292 +--
.../LogsearchKRBAuthenticationFilter.java | 1 -
.../LogsearchAuthenticationProvider.java | 4 +-
...rchExternalServerAuthenticationProvider.java | 9 +-
.../LogsearchFileAuthenticationProvider.java | 4 +-
.../LogsearchLdapAuthenticationProvider.java | 4 +-
.../LogsearchSimpleAuthenticationProvider.java | 4 +-
.../src/main/resources/default.properties | 1 -
.../main/resources/templates/audit_log_txt.ftl | 42 +
.../resources/templates/service_log_txt.ftl | 36 +
.../scripts/collection_bases/VLogListBase.js | 2 +-
.../src/main/webapp/scripts/utils/ViewUtils.js | 2 +-
.../scripts/views/audit/AuditAggregatedView.js | 6 +-
.../scripts/views/dashboard/LogLevelBoxView.js | 2 +-
.../views/filter/CreateLogfeederFilterView.js | 6 +-
.../views/tabs/HierarchyTabLayoutView.js | 1 +
.../troubleshoot/TroubleShootLayoutView.js | 6 +-
.../webapp/templates/tabs/LogFileView_tmpl.html | 50 +-
...BaseServiceLogRequestQueryConverterTest.java | 78 +
.../logging/LoggingRequestHelperImpl.java | 2 +-
129 files changed, 3143 insertions(+), 5647 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 486a0ba..be58732 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -757,5 +757,15 @@
<artifactId>spring-data-solr</artifactId>
<version>${spring-data-solr.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.freemarker</groupId>
+ <artifactId>freemarker</artifactId>
+ <version>2.3.20</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
index 9682a3d..c476b9d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import javax.inject.Inject;
+import javax.inject.Named;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
@@ -33,12 +34,11 @@ import org.apache.log4j.Logger;
import org.glassfish.jersey.client.JerseyClient;
import org.glassfish.jersey.client.JerseyClientBuilder;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
-import org.springframework.stereotype.Component;
/**
* Layer to send REST request to External server using jersey client
*/
-@Component
+@Named
public class ExternalServerClient {
private static Logger LOG = Logger.getLogger(ExternalServerClient.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
index d9d3b86..4e23146 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
@@ -29,7 +29,7 @@ public class LogSearchConstants {
public static final String FATAL = "FATAL";
public static final String UNKNOWN = "UNKNOWN";
- public static final String[] SUPPORTED_LOG_LEVEL ={FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN};
+ public static final String[] SUPPORTED_LOG_LEVELS = {FATAL, ERROR, WARN, INFO, DEBUG, TRACE, UNKNOWN};
// Application Constants
public static final String HOST = "H";
@@ -46,40 +46,10 @@ public class LogSearchConstants {
public static final String COMPOSITE_KEY = "composite_filtername-username";
public static final String SHARE_NAME_LIST = "share_username_list";
- // SOLR Document Constants for ServiceLogs
- public static final String BUNDLE_ID = "bundle_id";
- public static final String LOGTIME = "logtime";
- public static final String SEQUNCE_ID = "seq_num";
- public static final String SOLR_COMPONENT = "type";
- public static final String SOLR_LOG_MESSAGE = "log_message";
- public static final String SOLR_KEY_LOG_MESSAGE = "key_log_message";
- public static final String SOLR_HOST = "host";
- public static final String SOLR_LEVEL = "level";
- public static final String SOLR_THREAD_NAME = "thread_name";
- public static final String SOLR_LOGGER_NAME = "logger_name";
- public static final String SOLR_FILE = "file";
- public static final String SOLR_LINE_NUMBER = "line_number";
- public static final String SOLR_PATH = "path";
-
- //SOLR Document Constant for audit log
- public static final String AUDIT_COMPONENT = "repo";
- public static final String AUDIT_EVTTIME = "evtTime";
- public static final String AUDIT_REQUEST_USER = "reqUser";
-
- // Operator's
- public static final String MINUS_OPERATOR = "-";
- public static final String NO_OPERATOR = "";
-
- //operation
- public static final String EXCLUDE_QUERY = "excludeQuery";
- public static final String INCLUDE_QUERY = "includeQuery";
-
// Seprator's
public static final String I_E_SEPRATOR = "\\|i\\:\\:e\\|";
//SUFFIX
- public static final String UI_SUFFIX = "@UI@";
- public static final String SOLR_SUFFIX = "@Solr@";
public static final String NGRAM_SUFFIX = "ngram_";
//Date Format for SOLR
@@ -96,35 +66,21 @@ public class LogSearchConstants {
// logfeeder
public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config";
- public static final String LIST_SEPARATOR = ",";
-
+
public static final String SORT = "sort";
public static final String FL = "fl";
//Facet Constant
public static final String FACET_FIELD = "facet.field";
- public static final String FACET_MINCOUNT = "facet.mincount";
- public static final String FACET_JSON_FIELD = "json.facet";
public static final String FACET_PIVOT = "facet.pivot";
public static final String FACET_PIVOT_MINCOUNT = "facet.pivot.mincount";
- public static final String FACET_DATE = "facet.date";
- public static final String FACET_DATE_START = "facet.date.start";
- public static final String FACET_DATE_END = "facet.date.end";
- public static final String FACET_DATE_GAP = "facet.date.gap";
- public static final String FACET_RANGE = "facet.range";
- public static final String FACET_RANGE_START = "facet.range.start";
- public static final String FACET_RANGE_END = "facet.range.end";
- public static final String FACET_RANGE_GAP = "facet.range.gap";
- public static final String FACET_GROUP = "group";
- public static final String FACET_GROUP_MAIN = "group.main";
- public static final String FACET_GROUP_FIELD = "group.field";
// Request params
- public static final String REQUEST_PARAM_QUERY = "q";
public static final String REQUEST_PARAM_XAXIS = "xAxis";
public static final String REQUEST_PARAM_YAXIS = "yAxis";
public static final String REQUEST_PARAM_STACK_BY = "stackBy";
public static final String REQUEST_PARAM_UNIT = "unit";
+ public static final String REQUEST_PARAM_TOP = "top";
public static final String REQUEST_PARAM_BUNDLE_ID = "bundle_id";
public static final String REQUEST_PARAM_START_INDEX = "startIndex";
public static final String REQUEST_PARAM_PAGE = "page";
@@ -151,7 +107,6 @@ public class LogSearchConstants {
public static final String REQUEST_PARAM_HOST_NAME = "host_name";
public static final String REQUEST_PARAM_COMPONENT_NAME = "component_name";
public static final String REQUEST_PARAM_FILE_NAME = "file_name";
- public static final String REQUEST_PARAM_DATE_RANGE_LABEL = "dateRangeLabel";
public static final String REQUEST_PARAM_KEYWORD = "find";
public static final String REQUEST_PARAM_SOURCE_LOG_ID = "sourceLogId";
public static final String REQUEST_PARAM_KEYWORD_TYPE = "keywordType";
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java
new file mode 100644
index 0000000..2e6cddb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.common;
+
+public enum LogType {
+ SERVICE("Service"),
+ AUDIT("Audit");
+
+ private String label;
+
+ private LogType(String label) {
+ this.label = label;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
index b279a83..82a09b2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
@@ -18,12 +18,16 @@
*/
package org.apache.ambari.logsearch.conf;
+import freemarker.template.TemplateException;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.context.support.ConversionServiceFactoryBean;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
+import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean;
+
+import java.io.IOException;
@Configuration
@ComponentScan("org.apache.ambari.logsearch")
@@ -42,4 +46,13 @@ public class ApplicationConfig {
return conversionServiceFactoryBean;
}
+ @Bean
+ public freemarker.template.Configuration freemarkerConfiguration() throws IOException, TemplateException {
+ FreeMarkerConfigurationFactoryBean factoryBean = new FreeMarkerConfigurationFactoryBean();
+ factoryBean.setPreferFileSystemAccess(false);
+ factoryBean.setTemplateLoaderPath("classpath:/templates");
+ factoryBean.afterPropertiesSet();
+ return factoryBean.getObject();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
index 7508fb1..4868409 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.conf;
+import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder;
import org.apache.commons.lang.StringUtils;
@@ -79,6 +80,16 @@ public class SolrConfig {
solrUserConfigPropsConfig.getCollection()));
}
+ @Bean(name = "serviceSolrFieldDao")
+ public SolrSchemaFieldDao serviceSolrFieldDao() {
+ return new SolrSchemaFieldDao();
+ }
+
+ @Bean(name = "auditSolrFieldDao")
+ public SolrSchemaFieldDao auditSolrFieldDao() {
+ return new SolrSchemaFieldDao();
+ }
+
private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) {
if (StringUtils.isNotEmpty(zookeeperConnectString)) {
CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString);
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
index 8d6a4da..959d6f4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
@@ -21,17 +21,15 @@ package org.apache.ambari.logsearch.dao;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
+import javax.inject.Named;
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
-import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.solr.core.SolrTemplate;
-import org.springframework.stereotype.Component;
-@Component
+@Named
public class AuditSolrDao extends SolrDaoBase {
private static final Logger LOG = Logger.getLogger(AuditSolrDao.class);
@@ -40,7 +38,7 @@ public class AuditSolrDao extends SolrDaoBase {
private SolrAuditLogPropsConfig solrAuditLogPropsConfig;
@Inject
- @Qualifier("auditSolrTemplate")
+ @Named("auditSolrTemplate")
private SolrTemplate auditSolrTemplate;
@Inject
@@ -50,6 +48,7 @@ public class AuditSolrDao extends SolrDaoBase {
private SolrCollectionDao solrCollectionDao;
@Inject
+ @Named("auditSolrFieldDao")
private SolrSchemaFieldDao solrSchemaFieldDao;
public AuditSolrDao() {
@@ -57,8 +56,8 @@ public class AuditSolrDao extends SolrDaoBase {
}
@Override
- public CloudSolrClient getSolrClient() {
- return (CloudSolrClient) auditSolrTemplate.getSolrClient();
+ public SolrTemplate getSolrTemplate() {
+ return auditSolrTemplate;
}
@PostConstruct
@@ -71,13 +70,17 @@ public class AuditSolrDao extends SolrDaoBase {
boolean createAlias = (aliasNameIn != null && !StringUtils.isBlank(rangerAuditCollection));
solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig);
if (createAlias) {
- solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig, this);
+ solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig);
} else {
- solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig, this);
+ solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig);
}
} catch (Exception e) {
LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" +
solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e);
}
}
+
+ public SolrSchemaFieldDao getSolrSchemaFieldDao() {
+ return solrSchemaFieldDao;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
index a6f5acf..0e1d57c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
@@ -21,16 +21,14 @@ package org.apache.ambari.logsearch.dao;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
+import javax.inject.Named;
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig;
-import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
import org.apache.log4j.Logger;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.solr.core.SolrTemplate;
-import org.springframework.stereotype.Component;
-@Component
+@Named
public class ServiceLogsSolrDao extends SolrDaoBase {
private static final Logger LOG = Logger.getLogger(ServiceLogsSolrDao.class);
@@ -42,10 +40,11 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
private SolrServiceLogPropsConfig solrServiceLogPropsConfig;
@Inject
- @Qualifier("serviceSolrTemplate")
+ @Named("serviceSolrTemplate")
private SolrTemplate serviceSolrTemplate;
@Inject
+ @Named("serviceSolrFieldDao")
private SolrSchemaFieldDao solrSchemaFieldDao;
public ServiceLogsSolrDao() {
@@ -53,8 +52,8 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
}
@Override
- public CloudSolrClient getSolrClient() {
- return (CloudSolrClient) serviceSolrTemplate.getSolrClient();
+ public SolrTemplate getSolrTemplate() {
+ return serviceSolrTemplate;
}
@PostConstruct
@@ -63,11 +62,15 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
try {
solrCollectionDao.checkSolrStatus(getSolrClient());
solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig);
- solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig, this);
+ solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig);
} catch (Exception e) {
LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl()
+ ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString()
+ ", collection=" + solrServiceLogPropsConfig.getCollection(), e);
}
}
+
+ public SolrSchemaFieldDao getSolrSchemaFieldDao() {
+ return solrSchemaFieldDao;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
index 81471d3..6389446 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
@@ -26,16 +26,16 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import javax.inject.Named;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
-@Component
+@Named
public class SolrAliasDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class);
@@ -46,7 +46,7 @@ public class SolrAliasDao {
private SolrCollectionDao solrCollectionDao;
protected void setupAlias(final SolrSchemaFieldDao solrSchemaFieldDao, final CloudSolrClient solrClient,
- final SolrAuditLogPropsConfig solrPropsConfig, final SolrDaoBase solrDaoBase) throws Exception {
+ final SolrAuditLogPropsConfig solrPropsConfig) throws Exception {
final Collection<String> collectionListIn = Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) {
@@ -71,7 +71,7 @@ public class SolrAliasDao {
if (count == collectionListIn.size()) {
LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " +
"Collections=" + collectionListIn);
- solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig, solrDaoBase);
+ solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig);
break;
}
} else {
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
index ed93b4d..1cbac31 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
@@ -30,15 +30,15 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
+import javax.inject.Named;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
-@Component
+@Named
public class SolrCollectionDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 8381948..d9a9464 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -19,88 +19,91 @@
package org.apache.ambari.logsearch.dao;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.ambari.logsearch.common.LogSearchContext;
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
-import org.apache.ambari.logsearch.model.response.BarGraphData;
-import org.apache.ambari.logsearch.model.response.NameValueData;
-import org.apache.ambari.logsearch.util.DateUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
+import org.springframework.data.solr.core.DefaultQueryParser;
+import org.springframework.data.solr.core.SolrCallback;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.data.solr.core.query.SolrDataQuery;
+
+import java.io.IOException;
public abstract class SolrDaoBase {
+ private static final Logger LOG = Logger.getLogger(SolrDaoBase.class);
private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance");
- public Map<String, String> schemaFieldNameMap = new HashMap<>();
- public Map<String, String> schemaFieldTypeMap = new HashMap<>();
-
private LogType logType;
protected SolrDaoBase(LogType logType) {
this.logType = logType;
}
- public QueryResponse process(SolrQuery solrQuery) throws SolrServerException, IOException {
+ public QueryResponse process(SolrQuery solrQuery, String event) {
if (getSolrClient() != null) {
- String event = solrQuery.get("event");
+ event = event == null ? solrQuery.get("event") : event;
solrQuery.remove("event");
- QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST);
- if (event != null && !"/audit/logs/live/count".equalsIgnoreCase(event)) {
- LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " +
+ try {
+ QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST);
+ if (event != null) {
+ LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " +
solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " +
queryResponse.getElapsedTime());
+ }
+ return queryResponse;
+ } catch (Exception e){
+ LOG.error("Error during solrQuery=" + e);
+ throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
}
- return queryResponse;
} else {
throw RESTErrorUtil.createRESTException("Solr configuration improper for " + logType.getLabel() +" logs",
MessageEnums.ERROR_SYSTEM);
}
}
- @SuppressWarnings("unchecked")
- public void extractValuesFromBuckets(SimpleOrderedMap<Object> jsonFacetResponse, String outerField, String innerField,
- List<BarGraphData> histogramData) {
- NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse.get(outerField);
- ArrayList<Object> stackBuckets = (ArrayList<Object>) stack.get("buckets");
- for (Object temp : stackBuckets) {
- BarGraphData vBarGraphData = new BarGraphData();
-
- SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp;
- String name = ((String) level.getVal(0)).toUpperCase();
- vBarGraphData.setName(name);
+ public QueryResponse process(SolrQuery solrQuery) {
+ return process(solrQuery, null);
+ }
- Collection<NameValueData> vNameValues = new ArrayList<>();
- vBarGraphData.setDataCount(vNameValues);
- ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level.get(innerField)).get("buckets");
- for (Object temp1 : levelBuckets) {
- SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1;
- String value = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0));
+ public QueryResponse process(SolrDataQuery solrDataQuery) {
+ return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery));
+ }
- String count = "" + countValue.getVal(1);
- NameValueData vNameValue = new NameValueData();
- vNameValue.setName(value);
- vNameValue.setValue(count);
- vNameValues.add(vNameValue);
+ public long count(final SolrDataQuery solrDataQuery) {
+ return getSolrTemplate().execute(new SolrCallback<Long>() {
+ @Override
+ public Long doInSolr(SolrClient solrClient) throws SolrServerException, IOException {
+ SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(solrDataQuery);
+ solrQuery.setStart(0);
+ solrQuery.setRows(0);
+ QueryResponse queryResponse = solrClient.query(solrQuery);
+ long count = solrClient.query(solrQuery).getResults().getNumFound();
+ LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Count SolrQuery :- " +
+ solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " +
+ queryResponse.getElapsedTime() + " Count result :- " + count);
+ return count;
}
- histogramData.add(vBarGraphData);
- }
+ });
+ }
+
+ public QueryResponse process(SolrDataQuery solrDataQuery, String event) {
+ return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery), event);
}
- public abstract CloudSolrClient getSolrClient();
+ public CloudSolrClient getSolrClient() {
+ return (CloudSolrClient) getSolrTemplate().getSolrClient();
+ }
+
+ public abstract SolrTemplate getSolrTemplate();
+
+ public abstract SolrSchemaFieldDao getSolrSchemaFieldDao();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index b6764d0..f16dc41 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -33,16 +33,12 @@ import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-@Component
-@Scope(value = "prototype")
public class SolrSchemaFieldDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
@@ -52,11 +48,13 @@ public class SolrSchemaFieldDao {
private boolean populateFieldsThreadActive = false;
+ private Map<String, String> schemaFieldNameMap = new HashMap<>();
+ private Map<String, String> schemaFieldTypeMap = new HashMap<>();
+
@Inject
private SolrUserPropsConfig solrUserPropsConfig;
- public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig,
- final SolrDaoBase solrDao) {
+ public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) {
if (!populateFieldsThreadActive) {
populateFieldsThreadActive = true;
LOG.info("Creating thread to populated fields for collection=" + solrPropsConfig.getCollection());
@@ -69,7 +67,7 @@ public class SolrSchemaFieldDao {
try {
Thread.sleep(SETUP_RETRY_SECOND * 1000);
retryCount++;
- boolean _result = _populateSchemaFields(solrClient, solrPropsConfig, solrDao);
+ boolean _result = _populateSchemaFields(solrClient, solrPropsConfig);
if (_result) {
LOG.info("Populate fields for collection " + solrPropsConfig.getCollection() + " is success, Update it after " +
SETUP_UPDATE_SECOND + " sec");
@@ -94,7 +92,7 @@ public class SolrSchemaFieldDao {
/**
* Called from the thread. Don't call this directly
*/
- private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, SolrDaoBase solrDao) {
+ private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
SolrRequest<SchemaResponse> request = new SchemaRequest();
request.setMethod(SolrRequest.METHOD.GET);
request.setPath("/schema");
@@ -109,7 +107,7 @@ public class SolrSchemaFieldDao {
}
if (namedList != null) {
- extractSchemaFieldsName(namedList.toString(), solrDao.schemaFieldNameMap, solrDao.schemaFieldTypeMap);
+ extractSchemaFieldsName(namedList.toString(), schemaFieldNameMap, schemaFieldTypeMap);
return true;
}
}
@@ -156,4 +154,12 @@ public class SolrSchemaFieldDao {
LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM);
}
}
+
+ public Map<String, String> getSchemaFieldTypeMap() {
+ return schemaFieldTypeMap;
+ }
+
+ public Map<String, String> getSchemaFieldNameMap() {
+ return schemaFieldNameMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
index 88c4577..d650bb7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
@@ -28,14 +28,15 @@ import java.util.List;
import java.util.Scanner;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
+import javax.inject.Named;
import org.apache.ambari.logsearch.common.LogSearchContext;
+import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
import org.apache.ambari.logsearch.view.VLogfeederFilterWrapper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
@@ -47,15 +48,12 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import com.google.gson.JsonParseException;
-import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.log4j.Logger;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.solr.core.SolrTemplate;
-import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
-@Component
+@Named
public class UserConfigSolrDao extends SolrDaoBase {
private static final Logger LOG = Logger.getLogger(UserConfigSolrDao.class);
@@ -69,10 +67,11 @@ public class UserConfigSolrDao extends SolrDaoBase {
private SolrCollectionDao solrCollectionDao;
@Inject
+ @Named("serviceSolrFieldDao")
private SolrSchemaFieldDao solrSchemaFieldDao;
@Inject
- @Qualifier("userConfigSolrTemplate")
+ @Named("userConfigSolrTemplate")
private SolrTemplate userConfigSolrTemplate;
public UserConfigSolrDao() {
@@ -80,11 +79,10 @@ public class UserConfigSolrDao extends SolrDaoBase {
}
@Override
- public CloudSolrClient getSolrClient() {
- return (CloudSolrClient) userConfigSolrTemplate.getSolrClient();
+ public SolrTemplate getSolrTemplate() {
+ return userConfigSolrTemplate;
}
-
@PostConstruct
public void postConstructor() {
String solrUrl = solrUserConfig.getSolrUrl();
@@ -94,7 +92,7 @@ public class UserConfigSolrDao extends SolrDaoBase {
try {
solrCollectionDao.checkSolrStatus(getSolrClient());
solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
- solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig, this);
+ solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig);
intializeLogFeederFilter();
} catch (Exception e) {
@@ -201,6 +199,11 @@ public class UserConfigSolrDao extends SolrDaoBase {
return logfeederFilterWrapper;
}
+ @Override
+ public SolrSchemaFieldDao getSolrSchemaFieldDao() {
+ return solrSchemaFieldDao;
+ }
+
private String getHadoopServiceConfigJSON() {
StringBuilder result = new StringBuilder("");
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 0df2ca3..e6f5103 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -44,6 +44,7 @@ public class DocConstants {
public static final String IS_LAST_PAGE_D = "Show last page (true/false)";
public static final String FIELD_D = "Get values for particular field";
public static final String FORMAT_D = "File Export format, can be 'txt' or 'json'";
+ public static final String TOP = "Number that defines how many top element you would like to see.";
}
public class AuditOperationDescriptions {
@@ -51,11 +52,7 @@ public class DocConstants {
public static final String GET_AUDIT_LOGS_OD = "Get the list of logs details";
public static final String GET_AUDIT_COMPONENTS_OD = "Get the list of audit components currently active or having data in Solr";
public static final String GET_AUDIT_LINE_GRAPH_DATA_OD = "Get the data required for line graph";
- public static final String GET_TOP_AUDIT_USERS_OD = "Get the top audit users having maximum access";
- public static final String GET_TOP_AUDIT_RESOURCES_OD = "Get the top audit resources having maximum access";
- public static final String GET_LIVE_LOGS_COUNT_OD = "not required";
- public static final String GET_REQUEST_USER_LINE_GRAPH_OD = "not required";
- public static final String GET_ANY_GRAPH_DATA_OD = "Get the data generic enough to use for graph plots";
+ public static final String GET_TOP_AUDIT_RESOURCES_OD = "Get the top audit resource count (grouped by type)";
public static final String EXPORT_USER_TALBE_TO_TEXT_FILE_OD = "Export the tables shown on Audit tab";
public static final String GET_SERVICE_LOAD_OD = "The graph for showing the top users accessing the services";
}
@@ -64,7 +61,6 @@ public class DocConstants {
public static final String LEVEL_D = "filter for log level";
public static final String BUNDLE_ID = "filter for host";
public static final String FILE_NAME_D = "File name filter which is supported from browser url";
- public static final String DATE_RANGE_LABEL_D = "Date range label (e.g.: Today)";
public static final String HOST_NAME_D = "Host name filter which is supported from browser url";
public static final String COMPONENT_NAME_D = "Component name filter which is supported from browser url";
public static final String FIND_D = "Finding particular text on subsequent pages in case of table view with pagination";
@@ -72,7 +68,6 @@ public class DocConstants {
public static final String KEYWORD_TYPE_D = "Serching the find param value in previous or next in paginated table";
public static final String TOKEN_D = "unique number used along with FIND_D. The request can be canceled using this token";
public static final String SOURCE_LOG_ID_D = "fetch the record set having that log Id";
- public static final String G_MUST_NOT_D = "not required";
public static final String NUMBER_ROWS_D = "Getting rows after particular log entry - used in 'Preview' option";
public static final String SCROLL_TYPE_D = "Used in 'Preview' feature for getting records 'after' or 'before'";
public static final String UTC_OFFSET_D = "timezone offset";
@@ -86,18 +81,15 @@ public class DocConstants {
public static final String GET_LOG_LEVELS_COUNT_OD = "Get Log levels with their counts";
public static final String GET_COMPONENTS_COUNT_OD = "Get components with their counts";
public static final String GET_HOSTS_COUNT_OD = "Get hosts with their counts";
- public static final String GET_TREE_EXTENSION_OD = "Get host and compoenets hierarchy";
+ public static final String GET_TREE_EXTENSION_OD = "Get host and compoenets hierarchy with log counts";
public static final String GET_HISTOGRAM_DATA_OD = "Get data for histogram";
- public static final String CANCEL_FIND_REQUEST_OD = "Cancel the FIND_D param request using TOKEN_D";
public static final String EXPORT_TO_TEXT_FILE_OD = "Export the table data in file";
public static final String GET_COMPONENT_LIST_WITH_LEVEL_COUNT_OD = "Get components with log level distribution count";
- public static final String GET_EXTREME_DATES_FOR_BUNDLE_ID_OD = "Get the start and end time of particular bundle_id";
- public static final String GET_SERVICE_LOGS_FIELD_NAME_OD = "Get service logs schema fields name (Human readable)";
- public static final String GET_ANY_GRAPH_DATA_OD = "Get the data generic enough to use for graph plots";
- public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data";
+ public static final String GET_ANY_GRAPH_COUNT_DATA_OD = "Get the data generic enough to use for graph plots (yAzis is always count)";
public static final String GET_HOST_LIST_BY_COMPONENT_OD = "Get host list of components";
public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "Get service logs schema fields";
public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch";
+ public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data";
}
public class PublicOperationDescriptions {
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
index 287ff6c..85b5fda 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
@@ -19,324 +19,405 @@
package org.apache.ambari.logsearch.graph;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.common.MessageEnums;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.ambari.logsearch.model.response.BarGraphData;
import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse;
+import org.apache.ambari.logsearch.model.response.CountData;
+import org.apache.ambari.logsearch.model.response.CountDataListResponse;
+import org.apache.ambari.logsearch.model.response.GraphData;
+import org.apache.ambari.logsearch.model.response.GraphDataListResponse;
import org.apache.ambari.logsearch.model.response.NameValueData;
-import org.apache.ambari.logsearch.query.model.SearchCriteria;
-import org.apache.ambari.logsearch.dao.SolrDaoBase;
-import org.apache.ambari.logsearch.query.QueryGeneration;
-import org.apache.ambari.logsearch.util.RESTErrorUtil;
-import org.apache.ambari.logsearch.util.SolrUtil;
+import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
+import org.apache.ambari.logsearch.model.response.NodeData;
+import org.apache.ambari.logsearch.model.response.NodeListResponse;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
+import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.springframework.stereotype.Component;
+import org.apache.solr.common.util.NamedList;
-import javax.inject.Inject;
+import javax.inject.Named;
-@Component
-public class GraphDataGenerator extends GraphDataGeneratorBase {
+@Named
+public class GraphDataGenerator {
- private static final Logger logger = Logger.getLogger(GraphDataGenerator.class);
+ public BarGraphDataListResponse generateBarGraphDataResponseWithRanges(QueryResponse response, String typeField, boolean typeUppercase) {
+ BarGraphDataListResponse dataList = new BarGraphDataListResponse();
+ if (response == null) {
+ return dataList;
+ }
+ NamedList<List<PivotField>> facetPivotResponse = response.getFacetPivot();
+ if (response.getFacetPivot() == null) {
+ return dataList;
+ }
+ List<PivotField> pivotFields = facetPivotResponse.get(typeField);
+ for (int pivotIndex = 0; pivotIndex < pivotFields.size(); pivotIndex++) {
+ PivotField pivotField = facetPivotResponse.get(typeField).get(pivotIndex);
+ List<NameValueData> nameValues = generateNameValueDataList(pivotField.getFacetRanges());
+ BarGraphData barGraphData = new BarGraphData();
+ barGraphData.setDataCount(nameValues);
+ String typeValue = typeUppercase ? StringUtils.upperCase(pivotField.getValue().toString()) : pivotField.getValue().toString();
+ barGraphData.setName(typeValue);
+ dataList.getGraphData().add(barGraphData);
+ }
+ return dataList;
+ }
- @Inject
- private QueryGeneration queryGenerator;
+ public BarGraphDataListResponse generateSecondLevelBarGraphDataResponse(QueryResponse response, int val) {
+ BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse();
+ NamedList<List<PivotField>> pivotFieldNameList = response.getFacetPivot();
+ if (pivotFieldNameList == null) {
+ return barGraphDataListResponse;
+ }
+ List<PivotField> pivotFields = pivotFieldNameList.getVal(val);
+ List<BarGraphData> barGraphDataList = new ArrayList<>();
+ for (PivotField pivotField : pivotFields) {
+ BarGraphData barGraphData = new BarGraphData();
+ barGraphData.setName(String.valueOf(pivotField.getValue()));
+ List<PivotField> secondLevelPivotFields = pivotField.getPivot();
+ List<NameValueData> nameValueDataList = new ArrayList<>();
+ for (PivotField sPivotField : secondLevelPivotFields) {
+ NameValueData nvD = new NameValueData();
+ nvD.setName(String.valueOf(sPivotField.getValue()));
+ nvD.setValue(String.valueOf(sPivotField.getCount()));
+ nameValueDataList.add(nvD);
+ }
+ barGraphData.setDataCount(nameValueDataList);
+ barGraphDataList.add(barGraphData);
+ }
+ barGraphDataListResponse.setGraphData(barGraphDataList);
+ return barGraphDataListResponse;
+ }
- public BarGraphDataListResponse getAnyGraphData(SearchCriteria searchCriteria, SolrDaoBase solrDaoBase, SolrQuery solrQuery) {
- // X axis credentials
- String xAxisField = (String) searchCriteria.getParamValue("xAxis");
- String stackField = (String) searchCriteria.getParamValue("stackBy");
- String from = (String) searchCriteria.getParamValue("from");
- String to = (String) searchCriteria.getParamValue("to");
- String unit = (String) searchCriteria.getParamValue("unit");
- String typeXAxis = solrDaoBase.schemaFieldNameMap.get(xAxisField);
- typeXAxis = (StringUtils.isBlank(typeXAxis)) ? "string" : typeXAxis;
+ public BarGraphDataListResponse generateBarGraphFromFieldFacet(QueryResponse response, String facetField) {
+ BarGraphDataListResponse dataList = new BarGraphDataListResponse();
+ Collection<BarGraphData> vaDatas = new ArrayList<>();
+ dataList.setGraphData(vaDatas);
+ if (response == null) {
+ return dataList;
+ }
+ FacetField facetFieldObj = response.getFacetField(facetField);
+ if (facetFieldObj == null) {
+ return dataList;
+ }
+
+ List<Count> counts = facetFieldObj.getValues();
+ if (counts == null) {
+ return dataList;
+ }
+ for (Count cnt : counts) {
+ List<NameValueData> valueList = new ArrayList<>();
+ BarGraphData vBarGraphData = new BarGraphData();
+ vaDatas.add(vBarGraphData);
+ NameValueData vNameValue = new NameValueData();
+ vNameValue.setName(cnt.getName());
+ vBarGraphData.setName(cnt.getName().toUpperCase());
+ vNameValue.setValue("" + cnt.getCount());
+ valueList.add(vNameValue);
+ vBarGraphData.setDataCount(valueList);
+ }
+ return dataList;
+ }
+
+ public List<NameValueData> generateNameValueDataList(List<RangeFacet> rangeFacet) {
+ List<NameValueData> nameValues = new ArrayList<>();
+ if (rangeFacet == null) {
+ return nameValues;
+ }
+ RangeFacet range = rangeFacet.get(0);
- // Y axis credentials
- String yAxisField = (String) searchCriteria.getParamValue("yAxis");
- // add updated typeXAxis as a type parameter
- searchCriteria.addParam("type", typeXAxis);
- String fieldTime = (String) searchCriteria.getParamValue("fieldTime");
- // decide graph type based on user request parameter
- GraphType garphType = getGraphType(searchCriteria);
- switch (garphType) {
- case NORMAL_GRAPH:
- return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery);
- case RANGE_NON_STACK_GRAPH:
- return rangeNonStackGraph(xAxisField, yAxisField, from, to, unit, solrDaoBase, typeXAxis, fieldTime, solrQuery);
- case NON_RANGE_STACK_GRAPH:
- return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery);
- case RANGE_STACK_GRAPH:
- return rangeStackGraph(xAxisField, stackField, from, to, unit, solrDaoBase, solrQuery);
- default:
- logger.warn("Invalid graph type :" + garphType.name());
- return null;
+ if (range == null) {
+ return nameValues;
+ }
+ List<RangeFacet.Count> listCount = range.getCounts();
+ for (RangeFacet.Count cnt : listCount) {
+ NameValueData nameValue = new NameValueData();
+ nameValue.setName(String.valueOf(cnt.getValue()));
+ nameValue.setValue(String.valueOf(cnt.getCount()));
+ nameValues.add(nameValue);
}
+ return nameValues;
}
- private GraphType getGraphType(SearchCriteria searchCriteria) {
- // default graph type is unknown
- GraphType graphType = GraphType.UNKNOWN;
- // X axis credentials
- String xAxisField = (String) searchCriteria.getParamValue("xAxis");
- String stackField = (String) searchCriteria.getParamValue("stackBy");
- String from = (String) searchCriteria.getParamValue("from");
- String to = (String) searchCriteria.getParamValue("to");
- String xType = (String) searchCriteria.getParamValue("type");
- if (xType != null) {
- // Y axis credentials
- String yAxisField = (String) searchCriteria.getParamValue("yAxis");
- if (StringUtils.isBlank(xAxisField) || StringUtils.isBlank(yAxisField)) {
- graphType = GraphType.UNKNOWN;
- } else if (StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from)
- && !(xType.contains("date") || xType.contains("time"))) {
- graphType = GraphType.NORMAL_GRAPH;
- } else if (StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from)
- && (xType.contains("date") || xType.contains("time"))) {
- graphType = GraphType.RANGE_NON_STACK_GRAPH;
- } else if (!StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from)
- && !(xType.contains("date") || xType.contains("time"))) {
- graphType = GraphType.NON_RANGE_STACK_GRAPH;
- } else if (!StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from)
- && (xType.contains("date") || xType.contains("time"))) {
- graphType = GraphType.RANGE_STACK_GRAPH;
+ public List<Count> generateCount(QueryResponse response) {
+ List<Count> counts = new ArrayList<>();
+ List<FacetField> facetFields = null;
+ FacetField facetField = null;
+ if (response == null) {
+ return counts;
+ }
+
+ facetFields = response.getFacetFields();
+ if (facetFields == null) {
+ return counts;
+ }
+ if (!facetFields.isEmpty()) {
+ facetField = facetFields.get(0);
+ }
+ if (facetField != null) {
+ counts = facetField.getValues();
+ }
+ return counts;
+ }
+
+ public BarGraphDataListResponse getGraphDataWithDefaults(QueryResponse queryResponse, String field, String[] defaults) {
+ BarGraphDataListResponse response = new BarGraphDataListResponse();
+ BarGraphData barGraphData = new BarGraphData();
+ List<NameValueData> nameValues = generateLevelCountData(queryResponse, defaults);
+ barGraphData.setName(field);
+ barGraphData.setDataCount(nameValues);
+ response.setGraphData(Lists.newArrayList(barGraphData));
+ return response;
+ }
+
+ public NameValueDataListResponse getNameValueDataListResponseWithDefaults(QueryResponse response, String[] defaults) {
+ NameValueDataListResponse result = new NameValueDataListResponse();
+ result.setvNameValues(generateLevelCountData(response, defaults));
+ return result;
+ }
+
+ public NodeListResponse generateServiceNodeTreeFromFacetResponse(QueryResponse queryResponse,
+ String firstHierarchy, String secondHierarchy,
+ String firstType, String secondType) {
+ NodeListResponse response = new NodeListResponse();
+ if (queryResponse == null) {
+ return response;
+ }
+ NamedList<List<PivotField>> namedPivotFieldList = queryResponse.getFacetPivot();
+ List<PivotField> firstLevelPivots = namedPivotFieldList.get(firstHierarchy);
+ List<PivotField> secondLevelPivots = namedPivotFieldList.get(secondHierarchy);
+ if (!CollectionUtils.isNotEmpty(firstLevelPivots) || !CollectionUtils.isNotEmpty(secondLevelPivots)) {
+ return response;
+ }
+ List<NodeData> nodeDataList = buidTreeData(firstLevelPivots, secondLevelPivots, firstType, secondType);
+ response.setvNodeList(nodeDataList);
+ return response;
+ }
+
+ public NodeListResponse generateOneLevelServiceNodeTree(QueryResponse queryResponse, String componentLevelHirachy) {
+ NodeListResponse response = new NodeListResponse();
+ List<NodeData> datatList = new ArrayList<>();
+ List<List<PivotField>> listPivotField = new ArrayList<>();
+ NamedList<List<PivotField>> namedList = queryResponse.getFacetPivot();
+ if (namedList != null) {
+ listPivotField = namedList.getAll(componentLevelHirachy);
+ }
+ List<PivotField> secondHirarchicalPivotFields = null;
+ if (listPivotField == null || listPivotField.isEmpty()) {
+ return response;
+ } else {
+ secondHirarchicalPivotFields = listPivotField.get(0);
+ }
+ for (PivotField singlePivotField : secondHirarchicalPivotFields) {
+ if (singlePivotField != null) {
+ NodeData comp = new NodeData();
+ comp.setName("" + singlePivotField.getValue());
+ List<PivotField> levelList = singlePivotField.getPivot();
+ List<NameValueData> levelCountList = new ArrayList<>();
+ comp.setLogLevelCount(levelCountList);
+ if (levelList != null) {
+ for (PivotField levelPivot : levelList) {
+ NameValueData level = new NameValueData();
+ level.setName(("" + levelPivot.getValue()).toUpperCase());
+ level.setValue("" + levelPivot.getCount());
+ levelCountList.add(level);
+ }
+ }
+ datatList.add(comp);
}
}
- return graphType;
+ response.setvNodeList(datatList);
+ return response;
}
- @SuppressWarnings("unchecked")
- private BarGraphDataListResponse normalGraph(String xAxisField, String yAxisField, String from, String to, SolrDaoBase solrDaoBase,
- String typeXAxis, String fieldTime, SolrQuery solrQuery) {
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- Collection<BarGraphData> vBarGraphDatas = new ArrayList<BarGraphData>();
- BarGraphData vBarGraphData = new BarGraphData();
- Collection<NameValueData> vNameValues = new ArrayList<NameValueData>();
- SolrUtil.setMainQuery(solrQuery, null);
- queryGenerator.setSingleIncludeFilter(solrQuery, fieldTime, "[" + from + " TO " + to + "]");
- if (typeXAxis.contains("string") || typeXAxis.contains("key_lower_case") || typeXAxis.contains("text")) {
- SolrUtil.setFacetField(solrQuery, xAxisField);
- try {
- QueryResponse response = solrDaoBase.process(solrQuery);
- if (response != null && response.getResults() != null) {
- long count = response.getResults().getNumFound();
- if (count > 0) {
- FacetField facetField = response.getFacetField(xAxisField);
- if (facetField != null) {
- List<Count> countValues = facetField.getValues();
- if (countValues != null) {
- for (Count countValue : countValues) {
- if (countValue != null) {
- NameValueData vNameValue = new NameValueData();
- vNameValue.setName(countValue.getName());
- vNameValue.setValue("" + countValue.getCount());
- vNameValues.add(vNameValue);
- }
- }
+ private List<NodeData> buidTreeData(List<PivotField> firstHirarchicalPivotFields,
+ List<PivotField> secondHirarchicalPivotFields,
+ String firstPriority, String secondPriority) {
+ List<NodeData> extensionTree = new ArrayList<>();
+ if (firstHirarchicalPivotFields != null) {
+ for (PivotField pivotHost : firstHirarchicalPivotFields) {
+ if (pivotHost != null) {
+ NodeData hostNode = new NodeData();
+ String name = (pivotHost.getValue() == null ? "" : "" + pivotHost.getValue());
+ String value = "" + pivotHost.getCount();
+ if (!StringUtils.isBlank(name)) {
+ hostNode.setName(name);
+ }
+ if (!StringUtils.isBlank(value)) {
+ hostNode.setValue(value);
+ }
+ if (!StringUtils.isBlank(firstPriority)) {
+ hostNode.setType(firstPriority);
+ }
+
+ hostNode.setParent(true);
+ hostNode.setRoot(true);
+ PivotField hostPivot = null;
+ for (PivotField searchHost : secondHirarchicalPivotFields) {
+ if (!StringUtils.isBlank(hostNode.getName())
+ && hostNode.getName().equals(searchHost.getValue())) {
+ hostPivot = searchHost;
+ break;
+ }
+ }
+ List<PivotField> pivotLevelHost = hostPivot == null? null : hostPivot.getPivot();
+ if (pivotLevelHost != null) {
+ Collection<NameValueData> logLevelCount = new ArrayList<>();
+ for (PivotField pivotLevel : pivotLevelHost) {
+ if (pivotLevel != null) {
+ NameValueData vnameValue = new NameValueData();
+ String levelName = (pivotLevel.getValue() == null ? "" : "" + pivotLevel.getValue());
+ vnameValue.setName(levelName.toUpperCase());
+ vnameValue.setValue("" + pivotLevel.getCount());
+ logLevelCount.add(vnameValue);
}
- vBarGraphData.setName(xAxisField);
- vBarGraphDatas.add(vBarGraphData);
- dataList.setGraphData(vBarGraphDatas);
}
+ hostNode.setLogLevelCount(logLevelCount);
}
- }
- if (xAxisField.equalsIgnoreCase(LogSearchConstants.SOLR_LEVEL)) {
- Collection<NameValueData> sortedVNameValues = new ArrayList<NameValueData>();
- for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
- NameValueData value = new NameValueData();
- value.setName(level);
- String val = "0";
- for (NameValueData valueLevel : vNameValues) {
- if (valueLevel.getName().equalsIgnoreCase(level)) {
- val = valueLevel.getValue();
- break;
+ List<PivotField> pivotComponents = pivotHost.getPivot();
+ if (pivotComponents != null) {
+ Collection<NodeData> componentNodes = new ArrayList<>();
+ for (PivotField pivotComp : pivotComponents) {
+ if (pivotComp != null) {
+ NodeData compNode = new NodeData();
+ String compName = (pivotComp.getValue() == null ? "" : "" + pivotComp.getValue());
+ compNode.setName(compName);
+ if (!StringUtils.isBlank(secondPriority)) {
+ compNode.setType(secondPriority);
+ }
+ compNode.setValue("" + pivotComp.getCount());
+ compNode.setParent(false);
+ compNode.setRoot(false);
+ List<PivotField> pivotLevels = pivotComp.getPivot();
+ if (pivotLevels != null) {
+ Collection<NameValueData> logLevelCount = new ArrayList<>();
+ for (PivotField pivotLevel : pivotLevels) {
+ if (pivotLevel != null) {
+ NameValueData vnameValue = new NameValueData();
+ String compLevel = pivotLevel.getValue() == null ? "" : "" + pivotLevel.getValue();
+ vnameValue.setName((compLevel).toUpperCase());
+
+ vnameValue.setValue("" + pivotLevel.getCount());
+ logLevelCount.add(vnameValue);
+ }
+ }
+ compNode.setLogLevelCount(logLevelCount);
+ }
+ componentNodes.add(compNode);
}
}
- value.setValue(val);
- sortedVNameValues.add(value);
+ hostNode.setChilds(componentNodes);
}
- vBarGraphData.setDataCount(sortedVNameValues);
- } else {
- vBarGraphData.setDataCount(vNameValues);
- }
- return dataList;
- } catch (SolrException | SolrServerException | IOException e) {
- String query = solrQuery != null ? solrQuery.toQueryString() : "";
- logger.error("Got exception for solr query :" + query, e.getCause());
- }
- } else {
- SolrUtil.setRowCount(solrQuery, 0);
- String yAxis = yAxisField.contains("count") ? "sum" : yAxisField;
- String jsonQuery = queryGenerator.buildJSONFacetAggregatedFuncitonQuery(yAxis, xAxisField);
- SolrUtil.setJSONFacet(solrQuery, jsonQuery);
- try {
- QueryResponse response = solrDaoBase.process(solrQuery);
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
- if (jsonFacetResponse.toString().equals("{count=0}")) {
- return dataList;
+ extensionTree.add(hostNode);
}
- NameValueData value = new NameValueData();
- String sum = (String) jsonFacetResponse.getVal(1);
- value.setName(xAxisField);
- value.setValue(sum != null ? sum.substring(0, sum.indexOf(".")) : "");
- vNameValues.add(value);
- vBarGraphData.setDataCount(vNameValues);
- vBarGraphData.setName(xAxisField);
- vBarGraphDatas.add(vBarGraphData);
- dataList.setGraphData(vBarGraphDatas);
- return dataList;
- } catch (SolrException | SolrServerException | IOException e) {
- String query = solrQuery != null ? solrQuery.toQueryString() : "";
- logger.error("Got exception for solr query :" + query, e.getCause());
}
}
- return null;
+
+ return extensionTree;
}
- @SuppressWarnings("unchecked")
- private BarGraphDataListResponse nonRangeStackGraph(String xAxisField, String yAxisField, String stackField, String from, String to,
- SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) {
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- Collection<BarGraphData> vGraphData = new ArrayList<BarGraphData>();
- String mainQuery = queryGenerator.buildInclusiveRangeFilterQuery(fieldTime, from, to);
- SolrUtil.setMainQuery(solrQuery, mainQuery);
- SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
- String jsonQuery = "";
- if (SolrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
- String function = (yAxisField.contains("count")) ? "sum" : yAxisField;
- jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(stackField, xAxisField, function);
- } else {
- jsonQuery = queryGenerator.buildJsonFacetTermsRangeQuery(stackField, xAxisField);
- }
- try {
- SolrUtil.setJSONFacet(solrQuery, jsonQuery);
- dataList.setGraphData(vGraphData);
- QueryResponse response = solrDaoBase.process(solrQuery);
- if (response == null) {
- response = new QueryResponse();
- }
- Long count = response.getResults().getNumFound();
- if (count <= 0) {
- return dataList;
- }
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
- if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) {
- return dataList;
- }
- extractNonRangeStackValuesFromBucket(jsonFacetResponse, stackField, vGraphData, typeXAxis);
- if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField) && LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(xAxisField)) {
- Collection<BarGraphData> levelVGraphData = dataList.getGraphData();
- for (BarGraphData graphData : levelVGraphData) {
- Collection<NameValueData> valueList = graphData.getDataCount();
- Collection<NameValueData> valueListSorted = new ArrayList<NameValueData>();
- for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
- String val = "0";
- for (NameValueData value : valueList) {
- if (value.getName().equalsIgnoreCase(level)) {
- val = value.getValue();
- break;
- }
- }
- NameValueData v1 = new NameValueData();
- v1.setName(level.toUpperCase());
- v1.setValue(val);
- valueListSorted.add(v1);
- }
- graphData.setDataCount(valueListSorted);
- }
+ private List<NameValueData> generateLevelCountData(QueryResponse queryResponse, String[] defaults) {
+ List<NameValueData> nameValues = Lists.newLinkedList();
+ Map<String, NameValueData> linkedMap = Maps.newLinkedHashMap();
+ List<Count> counts = generateCount(queryResponse);
+ if (!CollectionUtils.isNotEmpty(counts)) {
+ return nameValues;
+ }
+ for (String defaultValue : defaults) {
+ NameValueData nameValue = new NameValueData();
+ nameValue.setName(defaultValue);
+ nameValue.setValue("0");
+ linkedMap.put(defaultValue, nameValue);
+ }
+ for (Count count : counts) {
+ if (!linkedMap.containsKey(count.getName())) {
+ NameValueData nameValue = new NameValueData();
+ String name = count.getName().toUpperCase();
+ nameValue.setName(name);
+ nameValue.setValue(String.valueOf(count.getCount()));
+ linkedMap.put(name, nameValue);
}
- return dataList;
- } catch (SolrException | IOException | SolrServerException e) {
- String query = solrQuery != null ? solrQuery.toQueryString() : "";
- logger.error("Got exception for solr query :" + query, e.getCause());
- throw RESTErrorUtil.createRESTException(MessageEnums.DATA_NOT_FOUND.getMessage().getMessage(), MessageEnums.DATA_NOT_FOUND);
}
+
+ for (Map.Entry<String, NameValueData> nameValueDataEntry : linkedMap.entrySet()) {
+ nameValues.add(nameValueDataEntry.getValue());
+ }
+ return nameValues;
}
- @SuppressWarnings("unchecked")
- private BarGraphDataListResponse rangeNonStackGraph(String xAxisField, String yAxisField, String from, String to, String unit,
- SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) {
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- Collection<BarGraphData> vBarGraphDatas = new ArrayList<BarGraphData>();
- BarGraphData vBarGraphData = new BarGraphData();
- Collection<NameValueData> vNameValues = new ArrayList<NameValueData>();
- SolrUtil.setMainQuery(solrQuery, null);
- if (SolrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
- queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to);
- return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery);
- } else {
- try {
- SolrUtil.setFacetRange(solrQuery, xAxisField, from, to, unit);
- QueryResponse response = solrDaoBase.process(solrQuery);
- if (response != null) {
- Long count = response.getResults().getNumFound();
- if (count > 0) {
- @SuppressWarnings("rawtypes")
- List<RangeFacet> rangeFacet = response.getFacetRanges();
- if (rangeFacet != null && rangeFacet.size() > 0) {
- List<RangeFacet.Count> listCount = rangeFacet.get(0).getCounts();
- if (listCount != null) {
- for (RangeFacet.Count cnt : listCount) {
- NameValueData vNameValue = new NameValueData();
- vNameValue.setName(cnt.getValue());
- vNameValue.setValue("" + cnt.getCount());
- vNameValues.add(vNameValue);
- }
- vBarGraphData.setDataCount(vNameValues);
- vBarGraphDatas.add(vBarGraphData);
- vBarGraphData.setName(xAxisField);
- dataList.setGraphData(vBarGraphDatas);
- }
- }
- }
- }
- return dataList;
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Got exception for solr query :" + solrQuery, e.getCause());
+ public CountDataListResponse generateCountResponseByField(QueryResponse response, String field) {
+ CountDataListResponse collection = new CountDataListResponse();
+ List<CountData> vCounts = new ArrayList<>();
+ if (response == null) {
+ return collection;
+ }
+ FacetField facetFields = response.getFacetField(field);
+ if (facetFields == null) {
+ return collection;
+ }
+ List<Count> fieldList = facetFields.getValues();
+
+ if (fieldList == null) {
+ return collection;
+ }
+
+ for (Count cnt : fieldList) {
+ if (cnt != null) {
+ CountData vCount = new CountData();
+ vCount.setName(cnt.getName());
+ vCount.setCount(cnt.getCount());
+ vCounts.add(vCount);
}
}
- return null;
+ collection.setvCounts(vCounts);
+ return collection;
}
- @SuppressWarnings("unchecked")
- private BarGraphDataListResponse rangeStackGraph(String xAxisField, String stackField, String from, String to, String unit,
- SolrDaoBase solrDaoBase, SolrQuery solrQuery) {
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- List<BarGraphData> histogramData = new ArrayList<BarGraphData>();
- SolrUtil.setMainQuery(solrQuery, null);
- SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
- String jsonHistogramQuery =
- queryGenerator.buildJSONFacetTermTimeRangeQuery(stackField, xAxisField, from, to, unit).replace("\\", "");
- try {
- solrQuery.set("json.facet", jsonHistogramQuery);
- SolrUtil.setRowCount(solrQuery, 0);
- QueryResponse response = solrDaoBase.process(solrQuery);
- if (response != null) {
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
- if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) {
- return dataList;
+ public GraphDataListResponse generateSimpleGraphResponse(QueryResponse response, String hierarchy) {
+ GraphDataListResponse graphInfo = new GraphDataListResponse();
+ if (response == null) {
+ return graphInfo;
+ }
+ List<List<PivotField>> hirarchicalPivotField = new ArrayList<List<PivotField>>();
+ List<GraphData> dataList = new ArrayList<>();
+ NamedList<List<PivotField>> namedList = response.getFacetPivot();
+ if (namedList != null) {
+ hirarchicalPivotField = namedList.getAll(hierarchy);
+ }
+ if (!hirarchicalPivotField.isEmpty()) {
+ dataList = buidGraphData(hirarchicalPivotField.get(0));
+ }
+ if (!dataList.isEmpty()) {
+ graphInfo.setGraphData(dataList);
+ }
+
+ return graphInfo;
+ }
+
+ private List<GraphData> buidGraphData(List<PivotField> pivotFields) {
+ List<GraphData> logList = new ArrayList<>();
+ if (pivotFields != null) {
+ for (PivotField pivotField : pivotFields) {
+ if (pivotField != null) {
+ GraphData logLevel = new GraphData();
+ logLevel.setName("" + pivotField.getValue());
+ logLevel.setCount(Long.valueOf(pivotField.getCount()));
+ if (pivotField.getPivot() != null) {
+ logLevel.setDataList(buidGraphData(pivotField.getPivot()));
+ }
+ logList.add(logLevel);
}
- extractRangeStackValuesFromBucket(jsonFacetResponse, "x", "y", histogramData);
- dataList.setGraphData(histogramData);
}
- return dataList;
- } catch (SolrException | IOException | SolrServerException e) {
- logger.error("Got exception for solr query :" + solrQuery, e.getCause());
}
- return null;
+ return logList;
}
}