You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/04/11 18:14:51 UTC
[37/51] [partial] ambari git commit: AMBARI-15679. Initial commit for
LogSearch module (oleewre)
http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java
new file mode 100644
index 0000000..c32dc36
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.graph;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.ambari.logsearch.util.DateUtil;
+import org.apache.ambari.logsearch.view.VBarGraphData;
+import org.apache.ambari.logsearch.view.VNameValue;
+import org.apache.solr.client.solrj.response.RangeFacet;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectWriter;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class GraphDataGeneratorBase {
+
+ @Autowired
+ DateUtil dateUtil;
+
+ @SuppressWarnings("unchecked")
+ protected void extractRangeStackValuesFromBucket(
+ SimpleOrderedMap<Object> jsonFacetResponse, String outerField,
+ String innerField, List<VBarGraphData> histogramData) {
+ NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse
+ .get(outerField);
+ ArrayList<Object> stackBuckets = (ArrayList<Object>) stack
+ .get("buckets");
+ for (Object temp : stackBuckets) {
+ VBarGraphData vBarGraphData = new VBarGraphData();
+
+ SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp;
+ String name = ((String) level.getVal(0)).toUpperCase();
+ vBarGraphData.setName(name);
+
+ Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
+ vBarGraphData.setDataCounts(vNameValues);
+ ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level
+ .get(innerField)).get("buckets");
+ for (Object temp1 : levelBuckets) {
+ SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1;
+ String value = dateUtil
+ .convertDateWithMillisecondsToSolrDate((Date) countValue
+ .getVal(0));
+
+ String count = "" + countValue.getVal(1);
+ VNameValue vNameValue = new VNameValue();
+ vNameValue.setName(value);
+ vNameValue.setValue(count);
+ vNameValues.add(vNameValue);
+ }
+ histogramData.add(vBarGraphData);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected boolean extractNonRangeStackValuesFromBucket(
+ SimpleOrderedMap<Object> jsonFacetResponse, String level,
+ Collection<VBarGraphData> vGraphDatas, String typeXAxis) {
+
+ boolean zeroFlag = true;
+ if (jsonFacetResponse.get(level).toString().equals("{count=0}")) {
+ return false;
+ }
+
+ NamedList<Object> list = (NamedList<Object>) jsonFacetResponse
+ .get(level);
+
+ ArrayList<Object> list3 = (ArrayList<Object>) list.get("buckets");
+ int i = 0;
+ for (i = 0; i < list3.size(); i++) {
+ VBarGraphData vGraphData = new VBarGraphData();
+
+
+ Collection<VNameValue> levelCounts = new ArrayList<VNameValue>();
+ vGraphData.setDataCounts(levelCounts);
+
+ SimpleOrderedMap<Object> valueCount = (SimpleOrderedMap<Object>) list3
+ .get(i);
+ String name = ("" + valueCount.getVal(0)).trim();
+ if (isTypeNumber(typeXAxis)) {
+ VNameValue nameValue = new VNameValue();
+ String value = ("" + valueCount.getVal(2)).trim().substring(0, ("" + valueCount.getVal(2)).indexOf("."));
+ nameValue.setName(name);
+ nameValue.setValue(value);
+ levelCounts.add(nameValue);
+ } else {
+ SimpleOrderedMap<Object> l1 = (SimpleOrderedMap<Object>) valueCount
+ .getVal(2);
+ ArrayList<Object> l2 = (ArrayList<Object>) l1.get("buckets");
+ for (int j = 0; l2 != null && j < l2.size(); j++) {
+ VNameValue nameValue = new VNameValue();
+ SimpleOrderedMap<Object> innerValueCount = (SimpleOrderedMap<Object>) l2
+ .get(j);
+ nameValue.setName(("" + innerValueCount.getVal(0)).trim());
+ nameValue.setValue(("" + innerValueCount.getVal(1)).trim());
+ levelCounts.add(nameValue);
+ }
+ }
+
+ vGraphData.setName(name);
+ vGraphDatas.add(vGraphData);
+ }
+ return zeroFlag;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected boolean extractValuesFromJson(
+ SimpleOrderedMap<Object> jsonFacetResponse, String level,
+ VBarGraphData histogramData, List<RangeFacet.Count> counts) {
+ histogramData.setName(level);
+ Collection<VNameValue> levelCounts = new ArrayList<VNameValue>();
+ histogramData.setDataCounts(levelCounts);
+ boolean zeroFlag = true;
+ if (jsonFacetResponse.get(level).toString().equals("{count=0}")) {
+ for (RangeFacet.Count date : counts) {
+ VNameValue nameValue = new VNameValue();
+
+ nameValue.setName(date.getValue());
+ nameValue.setValue("0");
+
+ levelCounts.add(nameValue);
+ }
+ return false;
+ }
+ NamedList<Object> list = (NamedList<Object>) jsonFacetResponse
+ .get(level);
+ NamedList<Object> list2 = (NamedList<Object>) list.getVal(1);
+ ArrayList<Object> list3 = (ArrayList<Object>) list2.get("buckets");
+ int i = 0;
+ for (RangeFacet.Count date : counts) {
+ VNameValue nameValue = new VNameValue();
+ SimpleOrderedMap<Object> valueCount = (SimpleOrderedMap<Object>) list3
+ .get(i);
+ String count = ("" + valueCount.getVal(1)).trim();
+ if (!"0".equals(count)) {
+ zeroFlag = false;
+ }
+ nameValue.setName(date.getValue());
+ nameValue.setValue(count);
+
+ levelCounts.add(nameValue);
+ i++;
+ }
+
+ return zeroFlag;
+ }
+
+ protected boolean isTypeNumber(String typeXAxis) {
+ return "long".contains(typeXAxis) || "int".contains(typeXAxis)
+ || "float".contains(typeXAxis) || "double".contains(typeXAxis);
+ }
+
+ public String convertObjToString(Object obj) throws IOException {
+ if (obj == null) {
+ return "";
+ }
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectWriter w = mapper.writerWithDefaultPrettyPrinter();
+ return w.writeValueAsString(obj);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java
new file mode 100644
index 0000000..44143f4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.graph;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
+import org.apache.ambari.logsearch.query.QueryGeneration;
+import org.apache.ambari.logsearch.util.ConfigUtil;
+import org.apache.ambari.logsearch.util.DateUtil;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.ambari.logsearch.util.StringUtil;
+import org.apache.ambari.logsearch.view.VBarDataList;
+import org.apache.ambari.logsearch.view.VBarGraphData;
+import org.apache.ambari.logsearch.view.VNameValue;
+import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.FacetField.Count;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.RangeFacet;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class GraphDataGnerator extends GraphDataGeneratorBase {
+
+ @Autowired
+ StringUtil stringUtil;
+
+ @Autowired
+ QueryGeneration queryGenerator;
+
+ @Autowired
+ RESTErrorUtil restErrorUtil;
+
+ @Autowired
+ DateUtil dateUtil;
+
+ private static Logger logger = Logger.getLogger(GraphDataGnerator.class);
+
+ public String getAnyGraphData(SearchCriteria searchCriteria,
+ SolrDaoBase solrDaoBase, SolrQuery solrQuery) {
+ // X axis credentials
+ String xAxisField = (String) searchCriteria.getParamValue("xAxis");
+ String stackField = (String) searchCriteria.getParamValue("stackBy");
+ String from = (String) searchCriteria.getParamValue("from");
+ String to = (String) searchCriteria.getParamValue("to");
+ String unit = (String) searchCriteria.getParamValue("unit");
+ String suffix = (String) searchCriteria.getParamValue("suffix");
+ String typeXAxis = ConfigUtil.schemaFieldsName.get(xAxisField + suffix);
+ typeXAxis = (stringUtil.isEmpty(typeXAxis)) ? "string" : typeXAxis;
+
+ // Y axis credentials
+ String yAxisField = (String) searchCriteria.getParamValue("yAxis");
+
+ searchCriteria.addParam("type", typeXAxis);
+ String fieldTime = (String) searchCriteria.getParamValue("feildTime");
+
+ int garphType = getGraphType(searchCriteria);
+
+ switch (garphType) {
+ case 1:
+ return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase,
+ typeXAxis, fieldTime, solrQuery);
+ case 2:
+ return rangeNonStackGraph(xAxisField, yAxisField, from, to, unit,
+ solrDaoBase, typeXAxis, fieldTime, solrQuery);
+ case 3:
+ return nonRangeStackGraph(xAxisField, yAxisField, stackField, from,
+ to, solrDaoBase, typeXAxis, fieldTime, solrQuery);
+ case 4:
+ return rangeStackGraph(xAxisField, yAxisField, stackField, from,
+ to, unit, solrDaoBase, typeXAxis, fieldTime, solrQuery);
+ default:
+ return null;
+ }
+ }
+
+ private int getGraphType(SearchCriteria searchCriteria) {
+ // X axis credentials
+ String xAxisField = (String) searchCriteria.getParamValue("xAxis");
+ String stackField = (String) searchCriteria.getParamValue("stackBy");
+ String from = (String) searchCriteria.getParamValue("from");
+ String to = (String) searchCriteria.getParamValue("to");
+ String xType = (String) searchCriteria.getParamValue("type");
+
+ if (xType == null)
+ return 0;
+
+ // Y axis credentials
+ String yAxisField = (String) searchCriteria.getParamValue("yAxis");
+ if (stringUtil.isEmpty(xAxisField) || stringUtil.isEmpty(yAxisField)) {
+ }
+ // Normal Graph Type
+ else if (stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to)
+ && !stringUtil.isEmpty(from)
+ && !(xType.contains("date") || xType.contains("time")))
+ return 1;
+ // Range(Non-Stack) Graph Type
+ else if (stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to)
+ && !stringUtil.isEmpty(from)
+ && (xType.contains("date") || xType.contains("time")))
+ return 2;
+ // Non-Range Stack Graph Type
+ else if (!stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to)
+ && !stringUtil.isEmpty(from)
+ && !(xType.contains("date") || xType.contains("time")))
+ return 3;
+ // Range Stack GraphType
+ else if (!stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to)
+ && !stringUtil.isEmpty(from)
+ && (xType.contains("date") || xType.contains("time")))
+ return 4;
+
+ return 0;
+ }
+
+ @SuppressWarnings("unchecked")
+ private String normalGraph(String xAxisField, String yAxisField,
+ String from, String to, SolrDaoBase solrDaoBase, String typeXAxis,
+ String fieldTime, SolrQuery solrQuery) {
+ VBarDataList dataList = new VBarDataList();
+ Collection<VBarGraphData> vBarGraphDatas = new ArrayList<VBarGraphData>();
+ VBarGraphData vBarGraphData = new VBarGraphData();
+ Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
+
+ queryGenerator.setMainQuery(solrQuery, null);
+ queryGenerator.setSingleIncludeFilter(solrQuery, fieldTime, "[" + from
+ + " TO " + to + "]");
+ if (typeXAxis.contains("string")
+ || typeXAxis.contains("key_lower_case")
+ || typeXAxis.contains("text")) {
+ queryGenerator.setFacetField(solrQuery, xAxisField);
+ try {
+ QueryResponse response = solrDaoBase.process(solrQuery);
+ Long count = response.getResults().getNumFound();
+
+ if (response != null && count > 0) {
+ FacetField facetField = response.getFacetField(xAxisField);
+ if (facetField != null) {
+ List<Count> countValues = facetField.getValues();
+ for (Count cnt : countValues) {
+ VNameValue vNameValue = new VNameValue();
+ vNameValue.setName(cnt.getName());
+ vNameValue.setValue("" + cnt.getCount());
+ vNameValues.add(vNameValue);
+ }
+ vBarGraphData.setDataCounts(vNameValues);
+ vBarGraphData.setName(xAxisField);
+ vBarGraphDatas.add(vBarGraphData);
+ dataList.setGraphData(vBarGraphDatas);
+ }
+ }
+ return convertObjToString(dataList);
+ } catch (SolrException | SolrServerException | IOException e) {
+
+ }
+ } else {
+ queryGenerator.setRowCount(solrQuery, 0);
+ String yAxis = yAxisField.contains("count") ? "sum" : yAxisField;
+ String jsonQuery = queryGenerator
+ .buildJSONFacetAggregatedFuncitonQuery(yAxis,
+ xAxisField);
+ queryGenerator.setJSONFacet(solrQuery, jsonQuery);
+ try {
+ QueryResponse response = solrDaoBase.process(solrQuery);
+
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response
+ .getResponse().get("facets");
+ if (jsonFacetResponse.toString().equals("{count=0}"))
+ return convertObjToString(dataList);
+ VNameValue value = new VNameValue();
+ String sum = jsonFacetResponse.getVal(1).toString();
+ value.setName(xAxisField);
+ value.setValue(sum.substring(0, sum.indexOf(".")));
+
+ vNameValues.add(value);
+ vBarGraphData.setDataCounts(vNameValues);
+ vBarGraphData.setName(xAxisField);
+ vBarGraphDatas.add(vBarGraphData);
+ dataList.setGraphData(vBarGraphDatas);
+ return convertObjToString(dataList);
+ } catch (SolrException | SolrServerException | IOException e) {
+
+ }
+ }
+
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ private String nonRangeStackGraph(String xAxisField, String yAxisField,
+ String stackField, String from, String to, SolrDaoBase solrDaoBase,
+ String typeXAxis, String fieldTime, SolrQuery solrQuery) {
+ VBarDataList dataList = new VBarDataList();
+ Collection<VBarGraphData> vGraphData = new ArrayList<VBarGraphData>();
+
+ String mainQuery = queryGenerator.buildInclusiveRangeFilterQuery(
+ fieldTime, from, to);
+ queryGenerator.setMainQuery(solrQuery, mainQuery);
+ queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
+ String jsonQuery = "";
+
+ if (isTypeNumber(typeXAxis)) {
+ String function = (yAxisField.contains("count")) ? "sum"
+ : yAxisField;
+ jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(
+ stackField, xAxisField, function);
+ } else {
+ jsonQuery = queryGenerator.buildJsonFacetTermsRangeQuery(
+ stackField, xAxisField);
+ }
+
+ try {
+ queryGenerator.setJSONFacet(solrQuery, jsonQuery);
+ dataList.setGraphData(vGraphData);
+
+ QueryResponse response = solrDaoBase.process(solrQuery);
+ if (response == null) {
+ response = new QueryResponse();
+ }
+ Long count = response.getResults().getNumFound();
+ if (count <= 0)
+ return convertObjToString(dataList);
+
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response
+ .getResponse().get("facets");
+ if (jsonFacetResponse == null
+ || jsonFacetResponse.toString().equals("{count=0}")) {
+ return convertObjToString(dataList);
+ }
+
+ extractNonRangeStackValuesFromBucket(jsonFacetResponse, stackField, vGraphData,
+ typeXAxis);
+
+ if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField)
+ && LogSearchConstants.SOLR_LEVEL
+ .equalsIgnoreCase(xAxisField)) {
+ Collection<VBarGraphData> levelVGraphData = dataList.getGraphData();
+ List<String> logLevels = new ArrayList<String>();
+
+ logLevels.add(LogSearchConstants.FATAL);
+ logLevels.add(LogSearchConstants.ERROR);
+ logLevels.add(LogSearchConstants.WARN);
+ logLevels.add(LogSearchConstants.INFO);
+ logLevels.add(LogSearchConstants.DEBUG);
+ logLevels.add(LogSearchConstants.TRACE);
+
+ for (VBarGraphData garphData : levelVGraphData) {
+ Collection<VNameValue> valueList = garphData.getDataCount();
+ Collection<VNameValue> valueListSorted = new ArrayList<VNameValue>();
+ for (String level : logLevels) {
+ String val = "0";
+ for (VNameValue value : valueList) {
+ if (value.getName().equalsIgnoreCase(level)) {
+ val = value.getValue();
+ break;
+ }
+ }
+ VNameValue v1 = new VNameValue();
+ v1.setName(level.toUpperCase());
+ v1.setValue(val);
+ valueListSorted.add(v1);
+ }
+ garphData.setDataCounts(valueListSorted);
+ }
+ }
+
+ return convertObjToString(dataList);
+ } catch (SolrException | IOException | SolrServerException e) {
+ logger.error(e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.DATA_NOT_FOUND);
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private String rangeNonStackGraph(String xAxisField, String yAxisField,
+ String from, String to, String unit, SolrDaoBase solrDaoBase,
+ String typeXAxis, String fieldTime, SolrQuery solrQuery) {
+ VBarDataList dataList = new VBarDataList();
+ Collection<VBarGraphData> vBarGraphDatas = new ArrayList<VBarGraphData>();
+ VBarGraphData vBarGraphData = new VBarGraphData();
+ Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
+
+ queryGenerator.setMainQuery(solrQuery, null);
+
+ if (isTypeNumber(typeXAxis)) {
+ queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to);
+ return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase,
+ typeXAxis, fieldTime, solrQuery);
+ } else {
+ try {
+ queryGenerator.setFacetRange(solrQuery, xAxisField, from, to,
+ unit);
+ QueryResponse response = solrDaoBase.process(solrQuery);
+ if (response == null)
+ response = new QueryResponse();
+ Long count = response.getResults().getNumFound();
+ if (count > 0) {
+
+ @SuppressWarnings("rawtypes")
+ List<RangeFacet> rangeFacet = response.getFacetRanges();
+ if (rangeFacet == null)
+ return convertObjToString(dataList);
+
+ List<RangeFacet.Count> listCount = rangeFacet.get(0)
+ .getCounts();
+ if (listCount != null) {
+ for (RangeFacet.Count cnt : listCount) {
+ VNameValue vNameValue = new VNameValue();
+ vNameValue.setName(cnt.getValue());
+ vNameValue.setValue("" + cnt.getCount());
+ vNameValues.add(vNameValue);
+ }
+ vBarGraphData.setDataCounts(vNameValues);
+ vBarGraphDatas.add(vBarGraphData);
+ vBarGraphData.setName(xAxisField);
+ dataList.setGraphData(vBarGraphDatas);
+ }
+ }
+ return convertObjToString(dataList);
+ } catch (SolrException | SolrServerException | IOException e) {
+
+ }
+ }
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ private String rangeStackGraph(String xAxisField, String yAxisField,
+ String stackField, String from, String to, String unit,
+ SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) {
+ VBarDataList dataList = new VBarDataList();
+ List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
+
+ queryGenerator.setMainQuery(solrQuery, null);
+ queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
+
+ String jsonHistogramQuery = queryGenerator
+ .buildJSONFacetTermTimeRangeQuery(stackField, xAxisField, from,
+ to, unit).replace("\\", "");
+
+ try {
+ solrQuery.set("json.facet", jsonHistogramQuery);
+ queryGenerator.setRowCount(solrQuery, 0);
+ QueryResponse response = solrDaoBase.process(solrQuery);
+ if (response == null)
+ response = new QueryResponse();
+
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response
+ .getResponse().get("facets");
+
+ if (jsonFacetResponse == null
+ || jsonFacetResponse.toString().equals("{count=0}"))
+ return convertObjToString(dataList);
+
+ extractRangeStackValuesFromBucket(jsonFacetResponse, "x", "y", histogramData);
+
+ dataList.setGraphData(histogramData);
+ return convertObjToString(dataList);
+
+ } catch (SolrException | IOException | SolrServerException e) {
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
new file mode 100644
index 0000000..e16faa0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
@@ -0,0 +1,810 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.manager;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.ManageStartEndTime;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.graph.GraphDataGnerator;
+import org.apache.ambari.logsearch.query.QueryGeneration;
+import org.apache.ambari.logsearch.util.BizUtil;
+import org.apache.ambari.logsearch.util.ConfigUtil;
+import org.apache.ambari.logsearch.util.DateUtil;
+import org.apache.ambari.logsearch.util.JSONUtil;
+import org.apache.ambari.logsearch.util.PropertiesUtil;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.ambari.logsearch.util.StringUtil;
+import org.apache.ambari.logsearch.view.VBarDataList;
+import org.apache.ambari.logsearch.view.VBarGraphData;
+import org.apache.ambari.logsearch.view.VGroupList;
+import org.apache.ambari.logsearch.view.VNameValue;
+import org.apache.ambari.logsearch.view.VNameValueList;
+import org.apache.ambari.logsearch.view.VSolrLogList;
+import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.FacetField.Count;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.RangeFacet;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AuditMgr extends MgrBase {
+ static Logger logger = Logger.getLogger(AuditMgr.class);
+
+ @Autowired
+ QueryGeneration queryGenerator;
+
+ @Autowired
+ AuditSolrDao auditSolrDao;
+
+ @Autowired
+ RESTErrorUtil restErrorUtil;
+
+ @Autowired
+ JSONUtil jsonUtil;
+
+ @Autowired
+ StringUtil stringUtil;
+
+ @Autowired
+ BizUtil bizUtil;
+
+ @Autowired
+ DateUtil dateUtil;
+
+ @Autowired
+ GraphDataGnerator graphDataGnerator;
+
+ public String getAllSolrFields() {
+
+ Collection<String> fieldNames = new ArrayList<String>();
+ SolrQuery solrQuery = new SolrQuery();
+
+ int numberDocument = 0;
+ int size = 0;
+
+ try {
+ queryGenerator.setMainQuery(solrQuery, null);
+ queryGenerator.setRowCount(solrQuery, 10);
+
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.setSortBy(LogSearchConstants.AUDIT_EVTTIME);
+ searchCriteria.setSortType(SolrQuery.ORDER.asc.toString());
+
+ queryGenerator.setSortOrderDefaultServiceLog(solrQuery,
+ searchCriteria);
+ SolrDocumentList docList = auditSolrDao.process(solrQuery)
+ .getResults();
+
+ if (docList.size() > 0) {
+ for (int i = 0; i < 10; i++) {
+ if (docList.get(i).size() > size) {
+ size = docList.get(i).size();
+ numberDocument = i;
+ }
+ }
+ fieldNames = docList.get(numberDocument).getFieldNames();
+ return convertObjToString(fieldNames);
+ }
+ return convertObjToString(fieldNames);
+
+ } catch (SolrException | SolrServerException | IOException e) {
+ logger.error(e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public String getLogs(SearchCriteria searchCriteria) {
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ try {
+ QueryResponse response = auditSolrDao.process(solrQuery);
+ SolrDocumentList docList = response.getResults();
+ VSolrLogList collection = new VSolrLogList(docList);
+ collection.setStartIndex((int) docList.getStart());
+ collection.setTotalCount(docList.getNumFound());
+ collection.setPageSize(searchCriteria.getMaxRows());
+ return convertObjToString(collection);
+ } catch (SolrException | SolrServerException | IOException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public SolrDocumentList getComponents(SearchCriteria searchCriteria) {
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ try {
+ queryGenerator.setFacetField(solrQuery,
+ LogSearchConstants.AUDIT_COMPONENT);
+ queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
+ List<FacetField> facetFields = null;
+ List<Count> componentsCount = new ArrayList<Count>();
+ FacetField facetField = null;
+
+ QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+ if (queryResponse == null) {
+ queryResponse = new QueryResponse();
+ }
+
+ facetFields = queryResponse.getFacetFields();
+ if (facetFields == null) {
+ return new SolrDocumentList();
+ }
+ if (!facetFields.isEmpty()) {
+ facetField = facetFields.get(0);
+ }
+ if (facetField != null) {
+ componentsCount = facetField.getValues();
+ }
+ SolrDocumentList docList = new SolrDocumentList();
+ for (Count compnonet : componentsCount) {
+ SolrDocument solrDocument = new SolrDocument();
+ solrDocument.addField("type", compnonet.getName());
+ docList.add(solrDocument);
+ }
+ return docList;
+ } catch (SolrException | SolrServerException | IOException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public String getAuditComponents(SearchCriteria searchCriteria) {
+ VGroupList vGroupList = new VGroupList();
+
+ try {
+
+ SolrDocumentList docList = getComponents(searchCriteria);
+
+ vGroupList.setGroupDocuments(docList);
+ return convertObjToString(vGroupList);
+ } catch (SolrException | IOException e) {
+ logger.error("Error", e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public String getAuditLineGraphData(SearchCriteria searchCriteria) {
+ VBarDataList dataList = new VBarDataList();
+ SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
+
+ String from = (String) searchCriteria.getParamValue("startTime");
+ String to = (String) searchCriteria.getParamValue("endTime");
+ String unit = (String) searchCriteria.getParamValue("unit");
+
+ List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
+ String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(
+ LogSearchConstants.AUDIT_COMPONENT,
+ LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\",
+ "");
+
+ try {
+ queryGenerator.setJSONFacet(solrQuery, jsonHistogramQuery);
+ queryGenerator.setRowCount(solrQuery, 0);
+ QueryResponse response = auditSolrDao.process(solrQuery);
+ if (response == null)
+ response = new QueryResponse();
+
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response
+ .getResponse().get("facets");
+
+ if (jsonFacetResponse == null
+ || jsonFacetResponse.toString().equals("{count=0}"))
+ return convertObjToString(dataList);
+
+ extractValuesFromBucket(jsonFacetResponse, "x", "y",
+ histogramData);
+
+ dataList.setGraphData(histogramData);
+ return convertObjToString(dataList);
+
+ } catch (SolrServerException | SolrException | IOException e) {
+ logger.error(e);
+ throw restErrorUtil.createRESTException("No Request Parameter",
+ MessageEnums.ERROR_SYSTEM);
+
+ }
+ }
+
+ public String getTopAuditFieldCount(SearchCriteria searchCriteria) {
+ Integer top = (Integer) searchCriteria.getParamValue("top");
+ String facetField = (String) searchCriteria.getParamValue("field");
+ if (top == null)
+ top = new Integer(10);
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ try {
+
+ List<VNameValue> nameValues = new ArrayList<VNameValue>();
+
+ VNameValueList nameValueList = new VNameValueList(nameValues);
+
+ queryGenerator.setFacetField(solrQuery, facetField);
+ queryGenerator.setFacetSort(solrQuery, "count");
+ queryGenerator.setFacetLimit(solrQuery, top.intValue());
+
+ List<Count> countList = new ArrayList<FacetField.Count>();
+ QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+ if (queryResponse.getFacetField(facetField) != null) {
+ FacetField queryFacetField = queryResponse
+ .getFacetField(facetField);
+ if (queryFacetField != null) {
+ countList = queryFacetField.getValues();
+ }
+ }
+
+ for (Count cnt : countList) {
+ VNameValue nameValue = new VNameValue();
+ nameValue.setName(cnt.getName());
+
+ nameValue.setValue("" + cnt.getCount());
+ nameValues.add(nameValue);
+ }
+ return convertObjToString(nameValueList);
+
+ } catch (SolrException | IOException | SolrServerException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public String getLiveLogCounts() {
+ SolrQuery solrQuery = new SolrQuery();
+ solrQuery.setParam("event", "/getLiveLogsCount");
+ try {
+ String startDate = dateUtil
+ .convertGivenDateFormatToSolrDateFormat(ManageStartEndTime.startDate);
+
+ String endDate = dateUtil
+ .convertGivenDateFormatToSolrDateFormat(ManageStartEndTime.endDate);
+
+ queryGenerator.setMainQuery(solrQuery, null);
+ queryGenerator.setFacetRange(solrQuery,
+ LogSearchConstants.AUDIT_EVTTIME, startDate, endDate,
+ "+2MINUTE");
+ List<RangeFacet.Count> listCount;
+
+ QueryResponse response = auditSolrDao.process(solrQuery);
+ @SuppressWarnings("rawtypes")
+ List<RangeFacet> rangeFacet = response.getFacetRanges();
+ if (rangeFacet == null)
+ return convertObjToString(new VNameValueList());
+ listCount = rangeFacet.get(0).getCounts();
+
+ List<VNameValue> nameValues = new ArrayList<VNameValue>();
+ int count = 0;
+ for (RangeFacet.Count cnt : listCount) {
+ VNameValue nameValue = new VNameValue();
+ nameValue.setName("" + count);
+ nameValue.setValue("" + cnt.getCount());
+ nameValues.add(nameValue);
+ count++;
+ }
+ VNameValueList nameValueList = new VNameValueList(nameValues);
+
+ return convertObjToString(nameValueList);
+
+ } catch (SolrException | SolrServerException | ParseException
+ | IOException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public String topTenUsers(SearchCriteria searchCriteria) {
+
+ String jsonUserQuery = "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}";
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ solrQuery.set("json.facet", jsonUserQuery);
+ queryGenerator.setRowCount(solrQuery, 0);
+ try {
+ QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+
+ NamedList<Object> namedList = queryResponse.getResponse();
+ VBarDataList vBarDataList = new VBarDataList();
+ if (namedList == null) {
+ return convertObjToString(vBarDataList);
+ }
+
+ @SuppressWarnings("unchecked")
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList
+ .get("facets");
+ vBarDataList = bizUtil.buildSummaryForTopCounts(jsonFacetResponse);
+ return convertObjToString(vBarDataList);
+
+ } catch (SolrServerException | SolrException | IOException e) {
+ logger.error("Error during solrQuery=" + e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public String topTenResources(SearchCriteria searchCriteria) {
+
+ String jsonUserQuery = "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}";
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ solrQuery.set("json.facet", jsonUserQuery);
+ queryGenerator.setRowCount(solrQuery, 0);
+ try {
+ QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+
+ NamedList<Object> namedList = queryResponse.getResponse();
+ VBarDataList vBarDataList = new VBarDataList();
+ if (namedList == null) {
+ return convertObjToString(vBarDataList);
+ }
+
+ @SuppressWarnings("unchecked")
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList
+ .get("facets");
+
+ vBarDataList = bizUtil.buildSummaryForTopCounts(jsonFacetResponse);
+ return convertObjToString(vBarDataList);
+
+ } catch (SolrServerException | SolrException | IOException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public String getRequestUserLineGraph(SearchCriteria searchCriteria) {
+
+ String from = (String) searchCriteria.getParamValue("startTime");
+ String to = (String) searchCriteria.getParamValue("endTime");
+ String unit = (String) searchCriteria.getParamValue("unit");
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+
+ VBarDataList dataList = new VBarDataList();
+ List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
+
+ queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
+
+ String jsonHistogramQuery = queryGenerator
+ .buildJSONFacetTermTimeRangeQuery(
+ LogSearchConstants.AUDIT_REQUEST_USER,
+ LogSearchConstants.AUDIT_EVTTIME, from, to, unit)
+ .replace("\\", "");
+
+ try {
+ solrQuery.set("json.facet", jsonHistogramQuery);
+ queryGenerator.setRowCount(solrQuery, 0);
+ QueryResponse response = auditSolrDao.process(solrQuery);
+ if (response == null)
+ response = new QueryResponse();
+
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response
+ .getResponse().get("facets");
+
+ if (jsonFacetResponse == null
+ || jsonFacetResponse.toString().equals("{count=0}"))
+ return convertObjToString(dataList);
+
+ extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData);
+
+ dataList.setGraphData(histogramData);
+ return convertObjToString(dataList);
+
+ } catch (SolrException | IOException | SolrServerException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+
+ }
+
+ public SolrDocumentList getRequestUser(SearchCriteria searchCriteria) {
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ try {
+ queryGenerator.setFacetField(solrQuery,
+ LogSearchConstants.AUDIT_REQUEST_USER);
+ queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
+ List<FacetField> facetFields = null;
+ List<Count> componentsCount = new ArrayList<Count>();
+ FacetField facetField = null;
+
+ QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+ if (queryResponse == null) {
+ queryResponse = new QueryResponse();
+ }
+
+ facetFields = queryResponse.getFacetFields();
+ if (facetFields == null) {
+ return new SolrDocumentList();
+ }
+ if (!facetFields.isEmpty()) {
+ facetField = facetFields.get(0);
+ }
+ if (facetField != null) {
+ componentsCount = facetField.getValues();
+ }
+ SolrDocumentList docList = new SolrDocumentList();
+ for (Count compnonet : componentsCount) {
+ SolrDocument solrDocument = new SolrDocument();
+ solrDocument.addField("type", compnonet.getName());
+ docList.add(solrDocument);
+ }
+ return docList;
+ } catch (SolrException | SolrServerException | IOException e) {
+ logger.error("Error during solrQuery=" + solrQuery, e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ public String getAuditLogsSchemaFieldsName() {
+ String suffix = PropertiesUtil.getProperty("auditlog.solr.core.logs");
+ String excludeArray[] = PropertiesUtil
+ .getPropertyStringList("auditlog.exclude.columnlist");
+ List<String> fieldNames = new ArrayList<String>();
+ HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>();
+ ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+
+ for (String fieldName : fieldNames) {
+ String uiField = ConfigUtil.auditLogsColumnMapping.get(fieldName
+ + LogSearchConstants.SOLR_SUFFIX);
+ if (uiField == null) {
+ uiFieldColumnMapping.put(fieldName, fieldName);
+ } else {
+ uiFieldColumnMapping.put(fieldName, uiField);
+ }
+ }
+
+ try {
+ uiFieldColumnMapping = bizUtil
+ .sortHashMapByValuesD(uiFieldColumnMapping);
+ return convertObjToString(uiFieldColumnMapping);
+ } catch (IOException e) {
+ logger.error(e);
+ }
+ throw restErrorUtil.createRESTException(
+ "Cache is Empty for FieldsName", MessageEnums.DATA_NOT_FOUND);
+ }
+
+ public String getAnyGraphData(SearchCriteria searchCriteria) {
+ searchCriteria.addParam("feildTime", LogSearchConstants.AUDIT_EVTTIME);
+ String suffix = PropertiesUtil.getProperty("auditlog.solr.core.logs");
+ searchCriteria.addParam("suffix", suffix);
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ String result = graphDataGnerator.getAnyGraphData(searchCriteria,
+ auditSolrDao, solrQuery);
+ if (result != null)
+ return result;
+ try {
+ return convertObjToString(new VBarDataList());
+ } catch (IOException e) {
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void extractValuesFromBucket(
+ SimpleOrderedMap<Object> jsonFacetResponse, String outerField,
+ String innerField, List<VBarGraphData> histogramData) {
+ NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse
+ .get(outerField);
+ ArrayList<Object> stackBuckets = (ArrayList<Object>) stack
+ .get("buckets");
+ for (Object temp : stackBuckets) {
+ VBarGraphData vBarGraphData = new VBarGraphData();
+
+ SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp;
+ String name = ((String) level.getVal(0)).toUpperCase();
+ vBarGraphData.setName(name);
+
+ Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
+ vBarGraphData.setDataCounts(vNameValues);
+ ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level
+ .get(innerField)).get("buckets");
+ for (Object temp1 : levelBuckets) {
+ SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1;
+ String value = dateUtil
+ .convertDateWithMillisecondsToSolrDate((Date) countValue
+ .getVal(0));
+
+ String count = "" + countValue.getVal(1);
+ VNameValue vNameValue = new VNameValue();
+ vNameValue.setName(value);
+ vNameValue.setValue(count);
+ vNameValues.add(vNameValue);
+ }
+ histogramData.add(vBarGraphData);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "resource"})
+ public Response exportUserTableToTextFile(SearchCriteria searchCriteria) {
+ String jsonUserQuery = "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}";
+
+ SolrQuery solrQuery = queryGenerator
+ .commonAuditFilterQuery(searchCriteria);
+ String startTime = (String) searchCriteria.getParamValue("startTime");
+ String endTime = (String) searchCriteria.getParamValue("endTime");
+
+ startTime = startTime == null ? "" : startTime;
+ endTime = endTime == null ? "" : "_" + endTime;
+
+ queryGenerator.setJSONFacet(solrQuery, jsonUserQuery);
+ queryGenerator.setRowCount(solrQuery, 0);
+
+ String dataFormat = (String) searchCriteria.getParamValue("format");
+ try {
+ QueryResponse queryResponse = auditSolrDao.process(solrQuery);
+
+ NamedList<Object> namedList = queryResponse.getResponse();
+ VBarDataList vBarUserDataList = new VBarDataList();
+ VBarDataList vBarResourceDataList = new VBarDataList();
+ if (namedList == null) {
+
+ }
+
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList
+ .get("facets");
+ vBarUserDataList = bizUtil
+ .buildSummaryForTopCounts(jsonFacetResponse);
+ vBarResourceDataList = bizUtil
+ .buildSummaryForResourceCounts(jsonFacetResponse);
+ String data = "";
+ String summary = "";
+ if ("text".equals(dataFormat)) {
+ int users = 0;
+ int resources = 0;
+ summary += "\n\n\n\n";
+ data += addBlank("Users") + "Components/Access" + "\n";
+ data += "--------------------------------------------------------------------------\n";
+ Collection<VBarGraphData> tableUserData = vBarUserDataList
+ .getGraphData();
+ for (VBarGraphData graphData : tableUserData) {
+ String userName = graphData.getName();
+ String largeUserName = "";
+
+ if (userName.length() > 45) {
+ largeUserName = userName.substring(0, 45);
+ data += addBlank(largeUserName);
+ } else
+ data += addBlank(userName);
+
+ Collection<VNameValue> vnameValueList = graphData
+ .getDataCount();
+ int count = 0;
+ String blank = "";
+ for (VNameValue vNameValue : vnameValueList) {
+ data += blank + vNameValue.getName() + " "
+ + vNameValue.getValue() + "\n";
+ if (count == 0)
+ blank = addBlank(blank);
+ count++;
+
+ }
+ while (largeUserName.length() > 0) {
+ data += largeUserName.substring(0, 45) + "\n";
+ }
+
+ users += 1;
+ }
+ data += "\n\n\n\n\n\n";
+ data += addBlank("Resources") + "Components/Access" + "\n";
+ data += "--------------------------------------------------------------------------\n";
+ Collection<VBarGraphData> tableResourceData = vBarResourceDataList
+ .getGraphData();
+ for (VBarGraphData graphData : tableResourceData) {
+ String resourceName = graphData.getName();
+ String largeResourceName = resourceName;
+ if (largeResourceName.length() > 45) {
+ resourceName = largeResourceName.substring(0, 45);
+ largeResourceName = largeResourceName.substring(45, largeResourceName.length());
+ } else {
+ largeResourceName = "";
+ }
+
+ //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n");
+ data += addBlank(resourceName);
+ Collection<VNameValue> vnameValueList = graphData
+ .getDataCount();
+ int count = 0;
+ String blank = "";
+ for (VNameValue vNameValue : vnameValueList) {
+ data += blank + vNameValue.getName() + " "
+ + vNameValue.getValue() + "\n";
+ if (count == 0)
+ blank = addBlank(blank);
+ count++;
+ }
+ String tempLargeResourceName = largeResourceName;
+ while (largeResourceName.length() > 45) {
+ largeResourceName = tempLargeResourceName.substring(0, 45);
+ tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length());
+ data += largeResourceName + "\n";
+ }
+ if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) {
+ data += largeResourceName + "\n";
+ }
+ resources += 1;
+ }
+ String header = "--------------------------------SUMMARY-----------------------------------\n";
+ summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary;
+ data = summary + data;
+ } else {
+ data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}";
+ dataFormat = "json";
+ }
+ String fileName = "Users_Resource" + startTime + endTime
+ + ".";
+ File file = File.createTempFile(fileName, dataFormat);
+
+ FileOutputStream fis = new FileOutputStream(file);
+ fis.write(data.getBytes());
+ return Response
+ .ok(file, MediaType.APPLICATION_OCTET_STREAM)
+ .header("Content-Disposition",
+ "attachment;filename=" + fileName + dataFormat)
+ .build();
+
+ } catch (SolrServerException | SolrException | IOException e) {
+ logger.error("Error during solrQuery=" + e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+ }
+
+ private String addBlank(String field) {
+ int blanks = 50;
+ int strSize = field.length();
+ String fieldWithBlank = field;
+ for (int i = 0; i < blanks - strSize; i++) {
+ fieldWithBlank += " ";
+ }
+ return fieldWithBlank;
+ }
+
+ public String getServiceLoad(SearchCriteria searchCriteria) {
+ VBarDataList dataList = new VBarDataList();
+ Collection<VBarGraphData> vaDatas = new ArrayList<VBarGraphData>();
+ dataList.setGraphData(vaDatas);
+
+ SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
+
+ try {
+ queryGenerator.setFacetField(serivceLoadQuery,
+ LogSearchConstants.AUDIT_COMPONENT);
+ QueryResponse serviceLoadResponse = auditSolrDao
+ .process(serivceLoadQuery);
+ if (serviceLoadResponse == null)
+ return "[]";
+ List<Count> serviceLoadFacets = serviceLoadResponse.getFacetField(
+ LogSearchConstants.AUDIT_COMPONENT).getValues();
+ for (Count cnt : serviceLoadFacets) {
+ List<VNameValue> valueList = new ArrayList<VNameValue>();
+ VBarGraphData vBarGraphData = new VBarGraphData();
+ vaDatas.add(vBarGraphData);
+ VNameValue vNameValue = new VNameValue();
+ vNameValue.setName(cnt.getName());
+ vBarGraphData.setName(cnt.getName().toUpperCase());
+ vNameValue.setValue("" + cnt.getCount());
+ valueList.add(vNameValue);
+ vBarGraphData.setDataCounts(valueList);
+ }
+
+
+ return convertObjToString(dataList);
+
+ } catch (SolrException | SolrServerException | IOException e) {
+ logger.error("Error during solrQuery=" + e);
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }
+
+ /*
+ String preDefinedJSON = getHadoopServiceConfigJSON();
+ try {
+ JSONObject serviceJSON = new JSONObject(preDefinedJSON).getJSONObject("service");
+ HashMap<String, Object> serviceMap = jsonUtil.jsonToMapObject(serviceJSON.toString());
+ Iterator<Entry<String, Object>> serviceMapIterator= serviceMap.entrySet().iterator();
+ List<VNameValue> newValueList = new ArrayList<VNameValue>();
+ for (VNameValue vNameValue : valueList) {
+ String name=vNameValue.getName();
+ while (serviceMapIterator.hasNext()) {
+ Map.Entry<String, Object> tempMap = serviceMapIterator
+ .next();
+
+ String keyName = tempMap.getKey();
+
+ JSONObject valueObj = new JSONObject(tempMap.toString().replace(keyName+"=", ""));
+ if(name.contains(keyName.toLowerCase())){
+ vNameValue.setName(valueObj.getString("label"));
+ break;
+ }
+ JSONArray componentsArray = valueObj.getJSONArray("components");
+
+ for(int i =0;i< componentsArray.length();i++){
+ JSONObject jObj = componentsArray.getJSONObject(i);
+ String jsonName = jObj.getString("name");
+ if(name.contains(jsonName.toLowerCase())){
+ vNameValue.setName(valueObj.getString("label"));
+ break;
+ }
+ }
+
+ }
+ if(newValueList.isEmpty()){
+ newValueList.add(vNameValue);
+ }else{
+ boolean isMatch = false;
+ for(VNameValue vValue: newValueList){
+ if(vValue.getName().equalsIgnoreCase(vNameValue.getName())){
+ isMatch =true;
+ Integer cnt1 = Integer.parseInt(vValue.getValue());
+ Integer cnt2 = Integer.parseInt(vNameValue.getValue());
+ vValue.setValue((cnt1+cnt2)+"");
+ }
+ }
+ if(!isMatch)
+ newValueList.add(vNameValue);
+ }
+ }
+ vBarGraphData.setDataCounts(newValueList);
+ vBarGraphData.setName("ServiceList");
+ return convertObjToString(dataList);
+
+ } catch (Exception e) {
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ }*/
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java
new file mode 100644
index 0000000..0388366
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.manager;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
+import org.apache.ambari.logsearch.view.VLogFile;
+import org.apache.ambari.logsearch.view.VLogFileList;
+import org.apache.ambari.logsearch.view.VSolrLogList;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.FacetField.Count;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+
+@Component
+public class LogFileMgr extends MgrBase {
+
+ private static Logger logger = Logger.getLogger(LogFileMgr.class);
+
+ private enum LOG_TYPE {
+ SERVICE, AUDIT
+ }
+
+ @Autowired
+ ServiceLogsSolrDao serviceLogsSolrDao;
+
+ @Autowired
+ AuditSolrDao auditSolrDao;
+
+ @Autowired
+ LogsMgr logMgr;
+
+ /**
+ * Search logFiles
+ *
+ * @param searchCriteria
+ * @return
+ */
+ public String searchLogFiles(SearchCriteria searchCriteria) {
+ VLogFileList logFileList = new VLogFileList();
+ List<VLogFile> logFiles = new ArrayList<VLogFile>();
+ String componentName = (String) searchCriteria.getParamValue("component");
+ String host = (String) searchCriteria.getParamValue("host");
+ int minCount = 1;// to remove zero count facet
+ SolrQuery solrQuery = new SolrQuery();
+ queryGenrator.setMainQuery(solrQuery, null);
+ queryGenrator.setFacetField(solrQuery, LogSearchConstants.SOLR_PATH, minCount);
+ // adding filter
+ queryGenrator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_COMPONENT, componentName);
+ queryGenrator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_HOST, host);
+ try {
+ String logType = (String) searchCriteria.getParamValue("logType");
+ if (stringUtil.isEmpty(logType)) {
+ logType = LOG_TYPE.SERVICE.name();// default is service Log
+ }
+ SolrDaoBase daoMgr = null;
+ if (logType.equalsIgnoreCase(LOG_TYPE.SERVICE.name())) {
+ daoMgr = serviceLogsSolrDao;
+ } else if (logType.equalsIgnoreCase(LOG_TYPE.AUDIT.name())) {
+ daoMgr = auditSolrDao;
+ } else {
+ throw restErrorUtil.createRESTException(logType + " is not a valid logType", MessageEnums.INVALID_INPUT_DATA);
+ }
+ QueryResponse queryResponse = daoMgr.process(solrQuery);
+ if (queryResponse.getFacetField(LogSearchConstants.SOLR_PATH) != null) {
+ FacetField queryFacetField = queryResponse.getFacetField(LogSearchConstants.SOLR_PATH);
+ if (queryFacetField != null) {
+ List<Count> countList = queryFacetField.getValues();
+ for (Count count : countList) {
+ VLogFile vLogFile = new VLogFile();
+ String filePath = count.getName();
+ String fileName = FilenameUtils.getName(filePath);
+ vLogFile.setPath(filePath);
+ vLogFile.setName(fileName);
+ logFiles.add(vLogFile);
+ }
+ }
+ }
+ } catch (SolrException | SolrServerException | IOException e) {
+ logger.error("Error in solr query :" + e.getLocalizedMessage() + "\n Query :" + solrQuery.toQueryString(),
+ e.getCause());
+ throw restErrorUtil.createRESTException("Please try later.", MessageEnums.ERROR_SYSTEM);
+ }
+ logFileList.setLogFiles(logFiles);
+ String jsonStr = "";
+ try {
+ jsonStr = convertObjToString(logFileList);
+ } catch (IOException e) {
+ logger.error(e);
+ throw restErrorUtil.createRESTException("Please try later.", MessageEnums.ERROR_SYSTEM);
+ }
+ return jsonStr;
+ }
+
+ public String getLogFileTail(SearchCriteria searchCriteria) {
+ String host = (String) searchCriteria.getParamValue("host");
+ String logFile = (String) searchCriteria.getParamValue("name");
+ String component = (String) searchCriteria.getParamValue("component");
+ String tailSize = (String) searchCriteria.getParamValue("tailSize");
+ if (stringUtil.isEmpty(host)) {
+ throw restErrorUtil.createRESTException("missing Host Name",
+ MessageEnums.ERROR_SYSTEM);
+ }
+ tailSize = (stringUtil.isEmpty(tailSize)) ? "10" : tailSize;
+ SolrQuery logFileTailQuery = new SolrQuery();
+ try {
+ int tail = Integer.parseInt(tailSize);
+ tail = tail > 100 ? 100 : tail;
+ queryGenrator.setMainQuery(logFileTailQuery, null);
+ queryGenrator.setSingleIncludeFilter(logFileTailQuery,
+ LogSearchConstants.SOLR_HOST, host);
+ if (!stringUtil.isEmpty(logFile)) {
+ queryGenrator.setSingleIncludeFilter(logFileTailQuery,
+ LogSearchConstants.SOLR_PATH,
+ solrUtil.makeSolrSearchString(logFile));
+ } else if (!stringUtil.isEmpty(component)) {
+ queryGenrator.setSingleIncludeFilter(logFileTailQuery,
+ LogSearchConstants.SOLR_COMPONENT, component);
+ } else {
+ throw restErrorUtil.createRESTException("component or logfile parameter must be present",
+ MessageEnums.ERROR_SYSTEM);
+ }
+
+ queryGenrator.setRowCount(logFileTailQuery, tail);
+ queryGenrator.setSortOrderDefaultServiceLog(logFileTailQuery, new SearchCriteria());
+ VSolrLogList solrLogList = getLogAsPaginationProvided(logFileTailQuery, serviceLogsSolrDao);
+ return convertObjToString(solrLogList);
+
+ } catch (SolrException | IOException e) {
+ throw restErrorUtil.createRESTException(e.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+ } catch (NumberFormatException ne) {
+
+ throw restErrorUtil.createRESTException(ne.getMessage(),
+ MessageEnums.ERROR_SYSTEM);
+
+ }
+ }
+}