You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/09/28 09:42:17 UTC
[50/52] [abbrv] ambari git commit: AMBARI-18310. Logsearch - Refactor
solr query layer (oleewere)
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java
deleted file mode 100644
index c57e0e9..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.graph;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.ambari.logsearch.manager.ManagerBase;
-import org.apache.ambari.logsearch.model.response.BarGraphData;
-import org.apache.ambari.logsearch.model.response.NameValueData;
-import org.apache.ambari.logsearch.util.DateUtil;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-
-class GraphDataGeneratorBase {
-
- private static final String BUCKETS = "buckets";
-
- private static enum DataType {
- LONG("long"),
- DOUBLE("double"),
- FLOAT("long"),
- INT("long");
-
- private String type;
-
- DataType(String type) {
- this.type = type;
- }
-
- String getType() {
- return type;
- }
- }
-
- protected static enum GraphType {
- UNKNOWN,
- NORMAL_GRAPH,
- RANGE_NON_STACK_GRAPH,
- NON_RANGE_STACK_GRAPH,
- RANGE_STACK_GRAPH;
- }
-
- @SuppressWarnings("unchecked")
- protected void extractRangeStackValuesFromBucket(SimpleOrderedMap<Object> jsonFacetResponse, String outerField,
- String innerField, List<BarGraphData> histogramData) {
- if (jsonFacetResponse != null) {
- NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse.get(outerField);
- if (stack != null) {
- ArrayList<Object> stackBuckets = (ArrayList<Object>) stack.get(BUCKETS);
- if (stackBuckets != null) {
- for (Object stackBucket : stackBuckets) {
- BarGraphData vBarGraphData = new BarGraphData();
- SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) stackBucket;
- if (level != null) {
- String name = level.getVal(0) != null ? level.getVal(0).toString().toUpperCase() : "";
- vBarGraphData.setName(name);
- Collection<NameValueData> vNameValues = new ArrayList<NameValueData>();
- NamedList<Object> innerFiledValue = (NamedList<Object>) level.get(innerField);
- if (innerFiledValue != null) {
- ArrayList<Object> levelBuckets = (ArrayList<Object>) innerFiledValue.get(BUCKETS);
- if (levelBuckets != null) {
- for (Object levelBucket : levelBuckets) {
- SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) levelBucket;
- if (countValue != null) {
- String innerName = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0));
- String innerValue = countValue.getVal(1) != null ? countValue.getVal(1).toString() : "";
- NameValueData vNameValue = new NameValueData();
- vNameValue.setName(innerName);
- vNameValue.setValue(innerValue);
- vNameValues.add(vNameValue);
- }
- }
- }
- }
- vBarGraphData.setDataCount(vNameValues);
- }
- histogramData.add(vBarGraphData);
- }
- }
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- protected boolean extractNonRangeStackValuesFromBucket(SimpleOrderedMap<Object> jsonFacetResponse, String level,
- Collection<BarGraphData> vGraphDatas, String typeXAxis) {
- boolean zeroFlag = true;
- if (jsonFacetResponse == null || jsonFacetResponse.get(level) == null
- || jsonFacetResponse.get(level).toString().equals("{count=0}")) {
- return false;
- }
- NamedList<Object> levelList = (NamedList<Object>) jsonFacetResponse.get(level);
- if (levelList != null) {
- ArrayList<Object> bucketList = (ArrayList<Object>) levelList.get(BUCKETS);
- if (bucketList != null) {
- for (int index = 0; index < bucketList.size(); index++) {
- SimpleOrderedMap<Object> valueCount = (SimpleOrderedMap<Object>) bucketList.get(index);
- if (valueCount != null && valueCount.size() > 2) {
- BarGraphData vGraphData = new BarGraphData();
- Collection<NameValueData> levelCounts = new ArrayList<NameValueData>();
- String name = valueCount.getVal(0) != null ? valueCount.getVal(0).toString().trim() : "";
- if (isTypeNumber(typeXAxis)) {
- NameValueData nameValue = new NameValueData();
- Double sumValue = (Double) valueCount.getVal(2);
- String value = "0";// default is zero
- if (sumValue != null) {
- value = "" + sumValue.longValue();
- }
- nameValue.setName(name);
- nameValue.setValue(value);
- levelCounts.add(nameValue);
- } else {
- SimpleOrderedMap<Object> valueCountMap = (SimpleOrderedMap<Object>) valueCount.getVal(2);
- if (valueCountMap != null) {
- ArrayList<Object> buckets = (ArrayList<Object>) valueCountMap.get(BUCKETS);
- if (buckets != null) {
- for (int innerIndex = 0; innerIndex < buckets.size(); innerIndex++) {
- SimpleOrderedMap<Object> innerValueCount = (SimpleOrderedMap<Object>) buckets.get(innerIndex);
- if (innerValueCount != null) {
- String innerName = innerValueCount.getVal(0) != null ? innerValueCount.getVal(0).toString().trim() : "";
- String innerValue = innerValueCount.getVal(1) != null ? innerValueCount.getVal(1).toString().trim() : "";
- NameValueData nameValue = new NameValueData();
- nameValue.setValue(innerValue);
- nameValue.setName(innerName);
- levelCounts.add(nameValue);
- }
- }
- }
- }
- }
- vGraphData.setName(name);
- vGraphData.setDataCount(levelCounts);
- vGraphDatas.add(vGraphData);
- }
- }
- }
- }
- return zeroFlag;
- }
-
- protected boolean isTypeNumber(String typeXAxis) {
- if (StringUtils.isBlank(typeXAxis)) {
- return false;
- } else {
- return typeXAxis.contains(DataType.LONG.getType()) || typeXAxis.contains(DataType.INT.getType())
- || typeXAxis.contains(DataType.FLOAT.getType()) || typeXAxis.contains(DataType.DOUBLE.getType());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
index 52482c2..56ba2d7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
@@ -20,398 +20,151 @@
package org.apache.ambari.logsearch.manager;
import javax.inject.Inject;
+import javax.inject.Named;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.text.ParseException;
+import java.io.StringWriter;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.common.ManageStartEndTime;
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+import freemarker.template.TemplateException;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
import org.apache.ambari.logsearch.dao.AuditSolrDao;
import org.apache.ambari.logsearch.graph.GraphDataGenerator;
+import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
+import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest;
+import org.apache.ambari.logsearch.model.request.impl.AuditLogRequest;
+import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest;
+import org.apache.ambari.logsearch.model.request.impl.FieldAuditLogRequest;
+import org.apache.ambari.logsearch.model.request.impl.UserExportRequest;
import org.apache.ambari.logsearch.model.response.AuditLogResponse;
import org.apache.ambari.logsearch.model.response.BarGraphData;
import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse;
import org.apache.ambari.logsearch.model.response.GroupListResponse;
import org.apache.ambari.logsearch.model.response.LogData;
-import org.apache.ambari.logsearch.model.response.LogSearchResponse;
import org.apache.ambari.logsearch.model.response.NameValueData;
-import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
-import org.apache.ambari.logsearch.query.model.AnyGraphSearchCriteria;
-import org.apache.ambari.logsearch.query.model.AuditBarGraphSearchCriteria;
-import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
-import org.apache.ambari.logsearch.query.model.FieldAuditBarGraphSearchCriteria;
-import org.apache.ambari.logsearch.query.model.FieldAuditLogSearchCriteria;
-import org.apache.ambari.logsearch.query.model.UserExportSearchCriteria;
+import org.apache.ambari.logsearch.model.response.TemplateData;
+import org.apache.ambari.logsearch.solr.SolrConstants;
import org.apache.ambari.logsearch.solr.model.SolrAuditLogData;
import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData;
-import org.apache.ambari.logsearch.util.BizUtil;
-import org.apache.ambari.logsearch.util.DateUtil;
+import org.apache.ambari.logsearch.util.DownloadUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
-import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.ambari.logsearch.view.VResponse;
-import org.apache.ambari.logsearch.query.model.AuditLogSearchCriteria;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.client.solrj.response.RangeFacet;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.springframework.stereotype.Component;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.data.solr.core.query.SimpleFacetQuery;
+import org.springframework.data.solr.core.query.SimpleQuery;
-@Component
+import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT;
+
+@Named
public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResponse> {
private static final Logger logger = Logger.getLogger(AuditLogsManager.class);
+ private static final String AUDIT_LOG_TEMPLATE = "audit_log_txt.ftl";
+
@Inject
private AuditSolrDao auditSolrDao;
@Inject
private GraphDataGenerator graphDataGenerator;
@Inject
- private SolrAuditLogPropsConfig solrAuditLogPropsConfig;
+ private ConversionService conversionService;
+ @Inject
+ private Configuration freemarkerConfiguration;
- public AuditLogResponse getLogs(AuditLogSearchCriteria searchCriteria) {
- Boolean isLastPage = searchCriteria.isLastPage();
- if (isLastPage) {
- SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- LogSearchResponse logResponse = getLastPage(searchCriteria, LogSearchConstants.AUDIT_EVTTIME, auditSolrDao, lastPageQuery);
- if (logResponse == null) {
- logResponse = new AuditLogResponse();
- }
- return (AuditLogResponse) logResponse;
- }
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- return getLogAsPaginationProvided(solrQuery, auditSolrDao);
+ public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) {
+ return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs");
}
- private List<LogData> getComponents(CommonSearchCriteria searchCriteria) {
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
+ private List<LogData> getComponents(AuditComponentRequest request) {
+ SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class);
List<LogData> docList = new ArrayList<>();
- try {
- SolrUtil.setFacetField(solrQuery, LogSearchConstants.AUDIT_COMPONENT);
- SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
- List<FacetField> facetFields = null;
- List<Count> componentsCount = new ArrayList<Count>();
- FacetField facetField = null;
+ QueryResponse queryResponse = auditSolrDao.process(facetQuery);
+ List<Count> componentsCount = graphDataGenerator.generateCount(queryResponse);
- QueryResponse queryResponse = auditSolrDao.process(solrQuery);
- if (queryResponse == null) {
- return docList;
- }
-
- facetFields = queryResponse.getFacetFields();
- if (facetFields == null) {
- return docList;
- }
- if (!facetFields.isEmpty()) {
- facetField = facetFields.get(0);
- }
- if (facetField != null) {
- componentsCount = facetField.getValues();
- }
-
- for (Count component : componentsCount) {
- SolrComponentTypeLogData logData = new SolrComponentTypeLogData();
- logData.setType(component.getName());
- docList.add(logData);
- }
- return docList;
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Error during solrQuery=" + solrQuery, e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
+ for (Count component : componentsCount) {
+ SolrComponentTypeLogData logData = new SolrComponentTypeLogData();
+ logData.setType(component.getName());
+ docList.add(logData);
}
+ return docList;
}
- public GroupListResponse getAuditComponents(CommonSearchCriteria searchCriteria) {
+ public GroupListResponse getAuditComponents(AuditComponentRequest request) {
GroupListResponse componentResponse = new GroupListResponse();
- List<LogData> docList = getComponents(searchCriteria);
+ List<LogData> docList = getComponents(request);
componentResponse.setGroupList(docList);
return componentResponse;
}
- @SuppressWarnings("unchecked")
- public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphSearchCriteria searchCriteria) {
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
-
- String from = getFrom(searchCriteria.getStartTime());
- String to = getTo(searchCriteria.getEndTime());
- String unit = getUnit(searchCriteria.getUnit());
-
- List<BarGraphData> histogramData = new ArrayList<BarGraphData>();
- String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT,
- LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", "");
-
- return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery);
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- public NameValueDataListResponse getLiveLogCounts() {
- NameValueDataListResponse nameValueList = new NameValueDataListResponse();
- SolrQuery solrQuery = new SolrQuery();
- solrQuery.setParam("event", "/audit/logs/live/count");
- try {
- Date[] timeRange = ManageStartEndTime.getStartEndTime();
- String startDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[0]);
- String endDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[1]);
-
- SolrUtil.setMainQuery(solrQuery, null);
- SolrUtil.setFacetRange(solrQuery, LogSearchConstants.AUDIT_EVTTIME, startDate, endDate, "+2MINUTE");
- List<RangeFacet.Count> listCount;
-
- QueryResponse response = auditSolrDao.process(solrQuery);
-
- List<RangeFacet> rangeFacet = response.getFacetRanges();
- if (rangeFacet == null) {
- return nameValueList;
- }
- RangeFacet range = rangeFacet.get(0);
-
- if (range == null) {
- return nameValueList;
- }
-
- listCount = range.getCounts();
-
- List<NameValueData> nameValues = new ArrayList<>();
- int count = 0;
- for (RangeFacet.Count cnt : listCount) {
- NameValueData nameValue = new NameValueData();
- nameValue.setName("" + count);
- nameValue.setValue("" + cnt.getCount());
- nameValues.add(nameValue);
- count++;
- }
- nameValueList.setvNameValues(nameValues);
- return nameValueList;
-
- } catch (SolrException | SolrServerException | ParseException
- | IOException e) {
- logger.error("Error during solrQuery=" + solrQuery, e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
- }
- }
-
- public BarGraphDataListResponse topTenUsers(FieldAuditBarGraphSearchCriteria searchCriteria) {
-
- String jsonUserQuery =
- "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}";
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- SolrUtil.setJSONFacet(solrQuery, jsonUserQuery);
- SolrUtil.setRowCount(solrQuery, 0);
- try {
- BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse();
- QueryResponse queryResponse = auditSolrDao.process(solrQuery);
- if (queryResponse == null) {
- return barGraphDataListResponse;
- }
-
- NamedList<Object> namedList = queryResponse.getResponse();
-
- if (namedList == null) {
- return barGraphDataListResponse;
- }
-
- @SuppressWarnings("unchecked")
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList.get("facets");
- if (jsonFacetResponse == null) {
- return barGraphDataListResponse;
- }
- if (jsonFacetResponse.toString().equals("{count=0}")) {
- return barGraphDataListResponse;
- }
- barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users");
- return barGraphDataListResponse;
-
- } catch (SolrServerException | SolrException | IOException e) {
- logger.error("Error during solrQuery=" + e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
- }
- }
-
- public BarGraphDataListResponse topTenResources(FieldAuditLogSearchCriteria searchCriteria) {
-
- String jsonUserQuery =
- "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}";
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- SolrUtil.setJSONFacet(solrQuery, jsonUserQuery);
- SolrUtil.setRowCount(solrQuery, 0);
- try {
- BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse();
- QueryResponse queryResponse = auditSolrDao.process(solrQuery);
- if (queryResponse == null) {
- return barGraphDataListResponse;
- }
-
- NamedList<Object> namedList = queryResponse.getResponse();
- if (namedList == null) {
- return barGraphDataListResponse;
- }
-
- @SuppressWarnings("unchecked")
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList.get("facets");
-
- barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users");
- return barGraphDataListResponse;
-
- } catch (SolrServerException | SolrException | IOException e) {
- logger.error("Error during solrQuery=" + solrQuery, e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
- }
+ public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphRequest request) {
+ SolrQuery solrQuery = conversionService.convert(request, SolrQuery.class);
+ QueryResponse response = auditSolrDao.process(solrQuery);
+ return graphDataGenerator.generateBarGraphDataResponseWithRanges(response, SolrConstants.AuditLogConstants.AUDIT_COMPONENT, true);
}
- @SuppressWarnings("unchecked")
- public BarGraphDataListResponse getRequestUserLineGraph(FieldAuditBarGraphSearchCriteria searchCriteria) {
-
- String from = getFrom(searchCriteria.getStartTime());
- String to = getTo(searchCriteria.getEndTime());
- String unit = getUnit(searchCriteria.getUnit());
-
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
-
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- List<BarGraphData> histogramData = new ArrayList<BarGraphData>();
-
- SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
-
- String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_REQUEST_USER,
- LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", "");
-
- return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery);
+ public BarGraphDataListResponse topResources(FieldAuditLogRequest request) {
+ SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class);
+ QueryResponse queryResponse = auditSolrDao.process(facetQuery);
+ return graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0);
}
public String getAuditLogsSchemaFieldsName() {
- return convertObjToString(auditSolrDao.schemaFieldNameMap);
+ return convertObjToString(auditSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap());
}
- public BarGraphDataListResponse getAnyGraphData(AnyGraphSearchCriteria searchCriteria) {
- searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME);
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery);
- if (result == null) {
- result = new BarGraphDataListResponse();
- }
- return result;
-
+ public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) {
+ SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class);
+ QueryResponse response = auditSolrDao.process(facetQuery);
+ return graphDataGenerator.generateBarGraphFromFieldFacet(response, AUDIT_COMPONENT);
}
- @SuppressWarnings({"unchecked"})
- public Response exportUserTableToTextFile(UserExportSearchCriteria searchCriteria) {
- String jsonUserQuery =
- "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}";
-
- SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- String startTime = searchCriteria.getStartTime();
- String endTime = searchCriteria.getEndTime();
+ public Response export(UserExportRequest request) throws TemplateException {
+ String startTime = request.getFrom();
+ String endTime = request.getTo();
+ SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class);
startTime = startTime == null ? "" : startTime;
endTime = endTime == null ? "" : "_" + endTime;
- SolrUtil.setJSONFacet(solrQuery, jsonUserQuery);
- SolrUtil.setRowCount(solrQuery, 0);
-
- String dataFormat = searchCriteria.getFormat();
+ String dataFormat = request.getFormat();
FileOutputStream fis = null;
try {
- QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+ QueryResponse queryResponse = auditSolrDao.process(facetQuery);
if (queryResponse == null) {
VResponse response = new VResponse();
- response.setMsgDesc("Query was not able to execute " + solrQuery);
+ response.setMsgDesc("Query was not able to execute " + facetQuery);
throw RESTErrorUtil.createRESTException(response);
}
-
- NamedList<Object> namedList = queryResponse.getResponse();
- if (namedList == null) {
- VResponse response = new VResponse();
- response.setMsgDesc("Query was not able to execute " + solrQuery);
- throw RESTErrorUtil.createRESTException(response);
- }
- BarGraphDataListResponse vBarUserDataList = new BarGraphDataListResponse();
- BarGraphDataListResponse vBarResourceDataList = new BarGraphDataListResponse();
-
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList.get("facets");
- vBarUserDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users");
- vBarResourceDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "y", "x");
+ BarGraphDataListResponse vBarUserDataList = graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0);
+ BarGraphDataListResponse vBarResourceDataList = graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 1);
String data = "";
- String summary = "";
if ("text".equals(dataFormat)) {
- int users = 0;
- int resources = 0;
- summary += "\n\n\n\n";
- data += addBlank("Users") + "Components/Access" + "\n";
- data += "--------------------------------------------------------------------------\n";
- Collection<BarGraphData> tableUserData = vBarUserDataList.getGraphData();
- for (BarGraphData graphData : tableUserData) {
- String userName = graphData.getName();
- String largeUserName = "";
-
- if (userName.length() > 45) {
- largeUserName = userName.substring(0, 45);
- data += addBlank(largeUserName);
- } else
- data += addBlank(userName);
-
- Collection<NameValueData> vnameValueList = graphData.getDataCount();
- data = appendNameValueData(data, vnameValueList);
- while (largeUserName.length() > 0) {
- data += largeUserName.substring(0, 45) + "\n";
- }
-
- users += 1;
- }
- data += "\n\n\n\n\n\n";
- data += addBlank("Resources") + "Components/Access" + "\n";
- data += "--------------------------------------------------------------------------\n";
- Collection<BarGraphData> tableResourceData = vBarResourceDataList.getGraphData();
- for (BarGraphData graphData : tableResourceData) {
- String resourceName = graphData.getName();
- String largeResourceName = resourceName;
- if (largeResourceName.length() > 45) {
- resourceName = largeResourceName.substring(0, 45);
- largeResourceName = largeResourceName.substring(45, largeResourceName.length());
- } else {
- largeResourceName = "";
- }
+ StringWriter stringWriter = new StringWriter();
+ Template template = freemarkerConfiguration.getTemplate(AUDIT_LOG_TEMPLATE);
+ Map<String, Object> models = new HashMap<>();
+ DownloadUtil.fillUserResourcesModel(models, vBarUserDataList, vBarResourceDataList);
+ template.process(models, stringWriter);
+ data = stringWriter.toString();
- //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n");
- data += addBlank(resourceName);
- Collection<NameValueData> vnameValueList = graphData.getDataCount();
- data = appendNameValueData(data, vnameValueList);
- String tempLargeResourceName = largeResourceName;
- while (largeResourceName.length() > 45) {
- largeResourceName = tempLargeResourceName.substring(0, 45);
- tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length());
- data += largeResourceName + "\n";
- }
- if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) {
- data += largeResourceName + "\n";
- }
- resources += 1;
- }
- String header = "--------------------------------SUMMARY-----------------------------------\n";
- summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary;
- data = summary + data;
} else {
data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}";
dataFormat = "json";
}
String fileName = "Users_Resource" + startTime + endTime + ".";
File file = File.createTempFile(fileName, dataFormat);
-
fis = new FileOutputStream(file);
fis.write(data.getBytes());
return Response
@@ -419,8 +172,8 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
.header("Content-Disposition", "attachment;filename=" + fileName + dataFormat)
.build();
- } catch (SolrServerException | SolrException | IOException e) {
- logger.error("Error during solrQuery=" + e);
+ } catch (IOException e) {
+ logger.error("Error during download file (audit log) " + e);
throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
} finally {
if (fis != null) {
@@ -432,97 +185,6 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
}
- private BarGraphDataListResponse getGraphResponse(BarGraphDataListResponse dataList, SolrQuery solrQuery,
- List<BarGraphData> histogramData, String jsonHistogramQuery) {
- try {
- SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery);
- SolrUtil.setRowCount(solrQuery, 0);
- QueryResponse response = auditSolrDao.process(solrQuery);
- if (response == null) {
- return dataList;
- }
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
-
- if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) {
- return dataList;
- }
-
- auditSolrDao.extractValuesFromBuckets(jsonFacetResponse, "x", "y", histogramData);
-
- dataList.setGraphData(histogramData);
- return dataList;
-
- } catch (SolrServerException | SolrException | IOException e) {
- logger.error(e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
-
- }
- }
-
- private String appendNameValueData(String data, Collection<NameValueData> vnameValueList) {
- int count = 0;
- String blank = "";
- for (NameValueData vNameValue : vnameValueList) {
- data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n";
- if (count == 0)
- blank = addBlank(blank);
- count++;
- }
- return data;
- }
-
- private String addBlank(String field) {
- int blanks = 50;
- int strSize = field.length();
- String fieldWithBlank = field;
- for (int i = 0; i < blanks - strSize; i++) {
- fieldWithBlank += " ";
- }
- return fieldWithBlank;
- }
-
- public BarGraphDataListResponse getServiceLoad(CommonSearchCriteria searchCriteria) {
- BarGraphDataListResponse dataList = new BarGraphDataListResponse();
- Collection<BarGraphData> vaDatas = new ArrayList<BarGraphData>();
- dataList.setGraphData(vaDatas);
-
- SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
-
- try {
- SolrUtil.setFacetField(serivceLoadQuery, LogSearchConstants.AUDIT_COMPONENT);
- QueryResponse serviceLoadResponse = auditSolrDao.process(serivceLoadQuery);
- if (serviceLoadResponse == null) {
- return dataList;
- }
- FacetField serviceFacetField = serviceLoadResponse.getFacetField(LogSearchConstants.AUDIT_COMPONENT);
- if (serviceFacetField == null) {
- return dataList;
- }
-
- List<Count> serviceLoadFacets = serviceFacetField.getValues();
- if (serviceLoadFacets == null) {
- return dataList;
- }
- for (Count cnt : serviceLoadFacets) {
- List<NameValueData> valueList = new ArrayList<NameValueData>();
- BarGraphData vBarGraphData = new BarGraphData();
- vaDatas.add(vBarGraphData);
- NameValueData vNameValue = new NameValueData();
- vNameValue.setName(cnt.getName());
- vBarGraphData.setName(cnt.getName().toUpperCase());
- vNameValue.setValue("" + cnt.getCount());
- valueList.add(vNameValue);
- vBarGraphData.setDataCount(valueList);
- }
-
- return dataList;
-
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Error during solrQuery=" + e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
- }
- }
-
@Override
protected List<SolrAuditLogData> convertToSolrBeans(QueryResponse response) {
return response.getBeans(SolrAuditLogData.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
index 13df470..a7b9cb5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java
@@ -28,17 +28,13 @@ import java.util.Date;
import java.util.List;
import java.util.Scanner;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.MessageEnums;
import org.apache.ambari.logsearch.model.response.LogData;
import org.apache.ambari.logsearch.model.response.LogSearchResponse;
-import org.apache.ambari.logsearch.query.model.SearchCriteria;
import org.apache.ambari.logsearch.dao.SolrDaoBase;
-import org.apache.ambari.logsearch.query.QueryGeneration;
import org.apache.ambari.logsearch.util.DateUtil;
import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
-import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
@@ -46,30 +42,13 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
-
-import javax.inject.Inject;
+import org.springframework.data.solr.core.DefaultQueryParser;
+import org.springframework.data.solr.core.query.SimpleQuery;
+import org.springframework.data.solr.core.query.SolrDataQuery;
public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE extends LogSearchResponse> extends JsonManagerBase {
private static final Logger logger = Logger.getLogger(ManagerBase.class);
- @Inject
- protected QueryGeneration queryGenerator;
-
- public enum LogType {
- SERVICE("Service"),
- AUDIT("Audit");
-
- private String label;
-
- private LogType(String label) {
- this.label = label;
- }
-
- public String getLabel() {
- return label;
- }
- }
-
public ManagerBase() {
super();
}
@@ -103,77 +82,52 @@ public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE
}
- protected SEARCH_RESPONSE getLastPage(SearchCriteria searchCriteria, String logTimeField, SolrDaoBase solrDoaBase,
- SolrQuery lastPageQuery) {
-
- Integer maxRows = searchCriteria.getMaxRows();
- String givenSortType = searchCriteria.getSortType();
- searchCriteria = new SearchCriteria();
- searchCriteria.setSortBy(logTimeField);
- if (givenSortType == null || givenSortType.equals(LogSearchConstants.DESCENDING_ORDER)) {
- lastPageQuery.removeSort(LogSearchConstants.LOGTIME);
- searchCriteria.setSortType(LogSearchConstants.ASCENDING_ORDER);
- } else {
- searchCriteria.setSortType(LogSearchConstants.DESCENDING_ORDER);
- }
- queryGenerator.setSingleSortOrder(lastPageQuery, searchCriteria);
-
-
- Long totalLogs = 0l;
- int startIndex = 0;
- int numberOfLogsOnLastPage = 0;
- SEARCH_RESPONSE logResponse = null;
- try {
- SolrUtil.setStart(lastPageQuery, 0);
- SolrUtil.setRowCount(lastPageQuery, maxRows);
- logResponse = getLogAsPaginationProvided(lastPageQuery, solrDoaBase);
- totalLogs = countQuery(lastPageQuery,solrDoaBase);
- startIndex = Integer.parseInt("" + ((totalLogs / maxRows) * maxRows));
- numberOfLogsOnLastPage = Integer.parseInt("" + (totalLogs - startIndex));
- logResponse.setStartIndex(startIndex);
- logResponse.setTotalCount(totalLogs);
- logResponse.setPageSize(maxRows);
- List<LOG_DATA_TYPE> docList = logResponse.getLogList();
- List<LOG_DATA_TYPE> lastPageDocList = new ArrayList<>();
- logResponse.setLogList(lastPageDocList);
- int cnt = 0;
- for(LOG_DATA_TYPE doc:docList){
- if(cnt<numberOfLogsOnLastPage){
- lastPageDocList.add(doc);
- }
- cnt++;
+ protected SEARCH_RESPONSE getLastPage(SolrDaoBase solrDoaBase, SimpleQuery lastPageQuery, String event) {
+ int maxRows = lastPageQuery.getRows();
+ SEARCH_RESPONSE logResponse = getLogAsPaginationProvided(lastPageQuery, solrDoaBase, event);
+ Long totalLogs = solrDoaBase.count(lastPageQuery);
+ int startIndex = Integer.parseInt("" + ((totalLogs / maxRows) * maxRows));
+ int numberOfLogsOnLastPage = Integer.parseInt("" + (totalLogs - startIndex));
+ logResponse.setStartIndex(startIndex);
+ logResponse.setTotalCount(totalLogs);
+ logResponse.setPageSize(maxRows);
+ List<LOG_DATA_TYPE> docList = logResponse.getLogList();
+ List<LOG_DATA_TYPE> lastPageDocList = new ArrayList<>();
+ logResponse.setLogList(lastPageDocList);
+ int cnt = 0;
+ for (LOG_DATA_TYPE doc : docList) {
+ if (cnt < numberOfLogsOnLastPage) {
+ lastPageDocList.add(doc);
}
- Collections.reverse(lastPageDocList);
-
- } catch (SolrException | SolrServerException | IOException | NumberFormatException e) {
- logger.error("Count Query was not executed successfully",e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
+ cnt++;
}
+ Collections.reverse(lastPageDocList);
return logResponse;
}
- protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase) {
- try {
- QueryResponse response = solrDaoBase.process(solrQuery);
- SEARCH_RESPONSE logResponse = createLogSearchResponse();
- SolrDocumentList docList = response.getResults();
- List<LOG_DATA_TYPE> serviceLogDataList = convertToSolrBeans(response);
- if (docList != null && !docList.isEmpty()) {
- logResponse.setLogList(serviceLogDataList);
- logResponse.setStartIndex((int) docList.getStart());
- logResponse.setTotalCount(docList.getNumFound());
- Integer rowNumber = solrQuery.getRows();
- if (rowNumber == null) {
- logger.error("No RowNumber was set in solrQuery");
- return createLogSearchResponse();
- }
- logResponse.setPageSize(rowNumber);
+ protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrDataQuery solrQuery, SolrDaoBase solrDaoBase, String event) {
+ SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(solrQuery);
+ return getLogAsPaginationProvided(query, solrDaoBase, event);
+ }
+
+
+ protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase, String event) {
+ QueryResponse response = solrDaoBase.process(solrQuery, event);
+ SEARCH_RESPONSE logResponse = createLogSearchResponse();
+ SolrDocumentList docList = response.getResults();
+ List<LOG_DATA_TYPE> serviceLogDataList = convertToSolrBeans(response);
+ if (docList != null && !docList.isEmpty()) {
+ logResponse.setLogList(serviceLogDataList);
+ logResponse.setStartIndex((int) docList.getStart());
+ logResponse.setTotalCount(docList.getNumFound());
+ Integer rowNumber = solrQuery.getRows();
+ if (rowNumber == null) {
+ logger.error("No RowNumber was set in solrQuery");
+ return createLogSearchResponse();
}
- return logResponse;
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Error during solrQuery=" + solrQuery, e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
+ logResponse.setPageSize(rowNumber);
}
+ return logResponse;
}
protected Long countQuery(SolrQuery query,SolrDaoBase solrDaoBase) throws SolrException, SolrServerException, IOException {
@@ -189,13 +143,6 @@ public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE
return docList.getNumFound();
}
- protected String getUnit(String unit) {
- if (StringUtils.isBlank(unit)) {
- unit = "+1HOUR";
- }
- return unit;
- }
-
protected String getFrom(String from) {
if (StringUtils.isBlank(from)) {
Date date = DateUtil.getTodayFromDate();
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
index 82817d6..cbab651 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
@@ -25,11 +25,11 @@ import java.util.List;
import org.apache.ambari.logsearch.conf.AuthPropsConfig;
import org.apache.ambari.logsearch.model.response.NameValueData;
import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
-import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import javax.inject.Named;
-@Component
+@Named
public class PublicManager extends JsonManagerBase {
@Inject