You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/09/28 09:41:34 UTC
[07/52] [abbrv] ambari git commit: AMBARI-18225. Fixed search issue
for number field in Logsearch portal (Hayat Behlim via oleewere)
AMBARI-18225. Fixed search issue for number field in Logsearch portal (Hayat Behlim via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/11984a06
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/11984a06
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/11984a06
Branch: refs/heads/logsearch-ga
Commit: 11984a0610794de65d4b7766aa3cdb979bdf7364
Parents: 90abd45
Author: oleewere <ol...@gmail.com>
Authored: Mon Aug 22 15:30:47 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Tue Sep 27 22:56:27 2016 +0200
----------------------------------------------------------------------
.../ambari-logsearch-portal/pom.xml | 21 ++++
.../logsearch/common/LogSearchConstants.java | 2 -
.../ambari/logsearch/dao/SolrDaoBase.java | 44 ++++----
.../apache/ambari/logsearch/dao/UserDao.java | 3 +
.../logsearch/graph/GraphDataGenerator.java | 22 ++--
.../ambari/logsearch/manager/AuditMgr.java | 6 +-
.../ambari/logsearch/manager/LogsMgr.java | 5 +-
.../ambari/logsearch/query/QueryGeneration.java | 103 ++++++++++++++-----
.../logsearch/query/QueryGenerationBase.java | 11 ++
.../ambari/logsearch/util/ConfigUtil.java | 70 +++++++------
.../apache/ambari/logsearch/util/SolrUtil.java | 54 ++++++++++
11 files changed, 243 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 05dc406..106c9b0 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -535,6 +535,27 @@
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>${solr.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
index 43a7eb7..5dcdac1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
@@ -91,8 +91,6 @@ public class LogSearchConstants {
public static final String UI_SUFFIX = "@UI@";
public static final String SOLR_SUFFIX = "@Solr@";
public static final String NGRAM_SUFFIX = "ngram_";
- public static final String DEFAULT_SERVICE_COLUMN_SUFFIX = "service";
- public static final String DEFAULT_AUDIT_COLUMN_SUFFIX = "audit";
//Date Format for SOLR
public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 91c4a26..cda5e26 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,10 +22,10 @@ package org.apache.ambari.logsearch.dao;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-
import org.apache.ambari.logsearch.common.LogsearchContextUtil;
import org.apache.ambari.logsearch.common.MessageEnums;
import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
@@ -60,7 +60,10 @@ import org.springframework.beans.factory.annotation.Autowired;
public abstract class SolrDaoBase {
static private Logger logger = Logger.getLogger(SolrDaoBase.class);
-
+
+ public HashMap<String, String> schemaFieldsNameMap = new HashMap<String, String>();
+ public HashMap<String, String> schemaFieldTypeMap = new HashMap<String, String>();
+
private static Logger logPerformance = Logger
.getLogger("org.apache.ambari.logsearch.performance");
@@ -95,6 +98,7 @@ public abstract class SolrDaoBase {
private boolean populateFieldsThreadActive = false;
int SETUP_RETRY_SECOND = 30;
+ int SETUP_UPDATE_SECOND = 10*60; //10 min
int ALIAS_SETUP_RETRY_SECOND = 30*60; //30 minutes
private boolean isZkConnectString=false;//by default its false
@@ -599,17 +603,16 @@ public abstract class SolrDaoBase {
}
private void populateSchemaFields() {
- boolean result = _populateSchemaFields();
- if (!result && !populateFieldsThreadActive) {
+ if (!populateFieldsThreadActive) {
populateFieldsThreadActive = true;
logger.info("Creating thread to populated fields for collection="
- + collectionName);
+ + collectionName);
Thread fieldPopulationThread = new Thread("populated_fields_"
- + collectionName) {
+ + collectionName) {
@Override
public void run() {
logger.info("Started thread to get fields for collection="
- + collectionName);
+ + collectionName);
int retryCount = 0;
while (true) {
try {
@@ -617,27 +620,25 @@ public abstract class SolrDaoBase {
retryCount++;
boolean _result = _populateSchemaFields();
if (_result) {
- logger.info("Populate fields for collection "
- + collectionName + " is success");
- break;
+ logger.info("Populate fields for collection " + collectionName
+ + " is success, Update it after " + SETUP_UPDATE_SECOND
+ + " sec");
+ Thread.sleep(SETUP_UPDATE_SECOND * 1000);
}
} catch (InterruptedException sleepInterrupted) {
- logger.info("Sleep interrupted while populating fields for collection "
- + collectionName);
+ logger
+ .info("Sleep interrupted while populating fields for collection "
+ + collectionName);
break;
} catch (Exception ex) {
logger.error("Error while populating fields for collection "
- + collectionName
- + ", retryCount="
- + retryCount);
- } finally {
- populateFieldsThreadActive = false;
+ + collectionName + ", retryCount=" + retryCount);
}
}
+ populateFieldsThreadActive = false;
logger.info("Exiting thread for populating fields. collection="
- + collectionName);
+ + collectionName);
}
-
};
fieldPopulationThread.setDaemon(true);
fieldPopulationThread.start();
@@ -650,7 +651,7 @@ public abstract class SolrDaoBase {
private boolean _populateSchemaFields() {
SolrRequest<SchemaResponse> request = new SchemaRequest();
request.setMethod(METHOD.GET);
- request.setPath("/schema/fields");
+ request.setPath("/schema");
String historyCollection = PropertiesUtil.getProperty("logsearch.solr.collection.history","history");
if (solrClient != null && !collectionName.equals(historyCollection)) {
NamedList<Object> namedList = null;
@@ -659,14 +660,13 @@ public abstract class SolrDaoBase {
logger.info("populateSchemaFields() collection="
+ collectionName + ", fields=" + namedList);
} catch (SolrException | SolrServerException | IOException e) {
-
logger.error(
"Error occured while popuplating field. collection="
+ collectionName, e);
}
if (namedList != null) {
ConfigUtil.extractSchemaFieldsName(namedList.toString(),
- collectionName);
+ schemaFieldsNameMap,schemaFieldTypeMap);
return true;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
index 091c40b..6b2f049 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
@@ -192,6 +192,9 @@ public class UserDao {
* @return
*/
public String encryptPassword(String username, String password) {
+ if (!stringUtil.isEmpty(username)) {
+ username = username.toLowerCase();
+ }
String saltEncodedpasswd = md5Encoder
.encodePassword(password, username);
return saltEncodedpasswd;
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
index 361f8e3..d3975b3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
@@ -32,6 +32,7 @@ import org.apache.ambari.logsearch.query.QueryGeneration;
import org.apache.ambari.logsearch.util.ConfigUtil;
import org.apache.ambari.logsearch.util.DateUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.ambari.logsearch.util.StringUtil;
import org.apache.ambari.logsearch.view.VBarDataList;
import org.apache.ambari.logsearch.view.VBarGraphData;
@@ -45,9 +46,12 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.TextField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+
+
@Component
public class GraphDataGenerator extends GraphDataGeneratorBase {
@@ -62,6 +66,9 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
@Autowired
DateUtil dateUtil;
+
+ @Autowired
+ SolrUtil solrUtil;
private static Logger logger = Logger.getLogger(GraphDataGenerator.class);
@@ -80,8 +87,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
String from = (String) searchCriteria.getParamValue("from");
String to = (String) searchCriteria.getParamValue("to");
String unit = (String) searchCriteria.getParamValue("unit");
- String suffix = (String) searchCriteria.getParamValue("suffix");
- String typeXAxis = ConfigUtil.schemaFieldsName.get(xAxisField + suffix);
+ String typeXAxis = solrDaoBase.schemaFieldsNameMap.get(xAxisField);
typeXAxis = (stringUtil.isEmpty(typeXAxis)) ? "string" : typeXAxis;
// Y axis credentials
@@ -102,8 +108,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, to,
solrDaoBase, typeXAxis, fieldTime, solrQuery);
case RANGE_STACK_GRAPH:
- return rangeStackGraph(xAxisField, yAxisField, stackField, from, to,
- unit, solrDaoBase, typeXAxis, fieldTime, solrQuery);
+ return rangeStackGraph(xAxisField, stackField, from, to, unit, solrDaoBase, solrQuery);
default:
logger.warn("Invalid graph type :" + garphType.name());
return null;
@@ -255,7 +260,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
queryGenerator.setMainQuery(solrQuery, mainQuery);
queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
String jsonQuery = "";
- if (isTypeNumber(typeXAxis)) {
+ if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
String function = (yAxisField.contains("count")) ? "sum" : yAxisField;
jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(stackField,
xAxisField, function);
@@ -323,7 +328,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
VBarGraphData vBarGraphData = new VBarGraphData();
Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
queryGenerator.setMainQuery(solrQuery, null);
- if (isTypeNumber(typeXAxis)) {
+ if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to);
return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase,
typeXAxis, fieldTime, solrQuery);
@@ -363,9 +368,8 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
}
@SuppressWarnings("unchecked")
- private VBarDataList rangeStackGraph(String xAxisField, String yAxisField,
- String stackField, String from, String to, String unit,
- SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime,
+ private VBarDataList rangeStackGraph(String xAxisField, String stackField,
+ String from, String to, String unit, SolrDaoBase solrDaoBase,
SolrQuery solrQuery) {
VBarDataList dataList = new VBarDataList();
List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
index 58c3a4d..ab287bc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
@@ -468,12 +468,11 @@ public class AuditMgr extends MgrBase {
}
public String getAuditLogsSchemaFieldsName() {
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
String excludeArray[] = PropertiesUtil
.getPropertyStringList("logsearch.solr.audit.logs.exclude.columnlist");
List<String> fieldNames = new ArrayList<String>();
HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>();
- ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+ ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,auditSolrDao);
for (String fieldName : fieldNames) {
String uiField = ConfigUtil.auditLogsColumnMapping.get(fieldName
@@ -492,15 +491,12 @@ public class AuditMgr extends MgrBase {
public String getAnyGraphData(SearchCriteria searchCriteria) {
searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME);
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
- searchCriteria.addParam("suffix", suffix);
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria,
auditSolrDao, solrQuery);
if (result == null) {
result = new VBarDataList();
}
-
return convertObjToString(result);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
index 748d2f9..257f04d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
@@ -1626,12 +1626,11 @@ public class LogsMgr extends MgrBase {
public String getServiceLogsSchemaFieldsName() {
List<String> fieldNames = new ArrayList<String>();
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
String excludeArray[] = PropertiesUtil
.getPropertyStringList("logsearch.solr.service.logs.exclude.columnlist");
HashMap<String, String> uiFieldColumnMapping = new LinkedHashMap<String, String>();
- ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+ ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,serviceLogsSolrDao);
for (String fieldName : fieldNames) {
String uiField = ConfigUtil.serviceLogsColumnMapping.get(fieldName
@@ -1695,8 +1694,6 @@ public class LogsMgr extends MgrBase {
public String getAnyGraphData(SearchCriteria searchCriteria) {
searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME);
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
- searchCriteria.addParam("suffix", suffix);
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria,
serviceLogsSolrDao, solrQuery);
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
index 646abd6..d934e84 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
@@ -27,11 +27,19 @@ import java.util.regex.Pattern;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
import org.apache.ambari.logsearch.util.ConfigUtil;
import org.apache.ambari.logsearch.util.PropertiesUtil;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
+import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.schema.TrieDoubleField;
+import org.apache.solr.schema.TrieFloatField;
+import org.apache.solr.schema.TrieLongField;
import org.springframework.stereotype.Component;
@Component
@@ -245,6 +253,10 @@ public class QueryGeneration extends QueryGenerationBase {
String value = getOriginalValue(originalKey,
"" + columnListMap.get(key));
orQuery = putWildCardByType(value, originalKey, logType);
+ if (stringUtil.isEmpty(orQuery)) {
+ logger.debug("Removing invalid filter for key :"+originalKey +" and value :" +value );
+ continue;
+ }
boolean isSame = false;
if (elments.contains(key)) {
isSame = true;
@@ -337,48 +349,72 @@ public class QueryGeneration extends QueryGenerationBase {
}
private String putWildCardByType(String str, String key, LOG_TYPE logType) {
- String type;
+ String fieldType;
+ SolrDaoBase solrDaoBase = null;
switch (logType) {
case AUDIT:
- String auditSuffix = PropertiesUtil
- .getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
- type = ConfigUtil.schemaFieldsName.get(key + auditSuffix);
+ fieldType = auditSolrDao.schemaFieldsNameMap.get(key);
+ solrDaoBase = auditSolrDao;
break;
case SERVICE:
- String serviceLogs = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
- type = ConfigUtil.schemaFieldsName.get(key + serviceLogs);
+ fieldType = serviceLogsSolrDao.schemaFieldsNameMap.get(key);
+ solrDaoBase = serviceLogsSolrDao;
if (key.equalsIgnoreCase(LogSearchConstants.SOLR_LOG_MESSAGE)) {
return solrUtil.escapeForLogMessage(key, str);
}
break;
default:
// set as null
- type = null;
+ logger.error("Invalid logtype :" + logType);
+ fieldType = null;
}
- if (type == null) {
- return key + ":" + "*" + str + "*";
- } else if ("text_std_token_lower_case".equalsIgnoreCase(type)) {
- return key + ":" + solrUtil.escapeForStandardTokenizer(str);
- } else if ("key_lower_case".equalsIgnoreCase(type)
- || "string".equalsIgnoreCase(type)) {
- return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str);
- } else if ("ip_address".equalsIgnoreCase(type)) {
- return key + ":" + str;
+ if (!stringUtil.isEmpty(fieldType)) {
+ if (solrUtil.isSolrFieldNumber(fieldType, solrDaoBase)) {
+ String value = putEscapeCharacterForNumber(str, fieldType,solrDaoBase);
+ if (!stringUtil.isEmpty(value)) {
+ return key + ":" + value;
+ } else {
+ return null;
+ }
+ } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class,solrDaoBase)) {
+ return key + ":" + solrUtil.escapeForStandardTokenizer(str);
+ } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class,solrDaoBase)|| "string".equalsIgnoreCase(fieldType)) {
+ return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str);
+ } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class,solrDaoBase)) {
+ return key + ":" + str;
+ }
}
- return key + ":" + putEscapeCharacterForNumber(str);
+ return key + ":" + "*" + str + "*";
}
- private String putEscapeCharacterForNumber(String str) {
- String escapeCharSting = "" + returnDefaultIfValueNotNumber(str);
- escapeCharSting = str.replace("-", "\\-");
+ private String putEscapeCharacterForNumber(String str,String fieldType,SolrDaoBase solrDaoBase) {
+ if (!stringUtil.isEmpty(str)) {
+ str = str.replace("*", "");
+ }
+ String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType,solrDaoBase);
+ if (escapeCharSting == null || escapeCharSting.isEmpty()) {
+ return null;
+ }
+ escapeCharSting = escapeCharSting.replace("-", "\\-");
return escapeCharSting;
}
- private String returnDefaultIfValueNotNumber(String str) {
+ private String parseInputValueAsPerFieldType(String str,String fieldType,SolrDaoBase solrDaoBase ) {
try {
- return "" + Integer.parseInt(str);
+ HashMap<String, Object> fieldTypeInfoMap= solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase);
+ String className = (String) fieldTypeInfoMap.get("class");
+ if( className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())){
+ return ""+ Double.parseDouble(str);
+ }else if(className.equalsIgnoreCase(TrieFloatField.class.getSimpleName())){
+ return ""+ Float.parseFloat(str);
+ }else if(className.equalsIgnoreCase(TrieLongField.class.getSimpleName())){
+ return ""+ Long.parseLong(str);
+ }else {
+ return "" + Integer.parseInt(str);
+ }
} catch (Exception e) {
- return "0";
+ logger.debug("Invaid input str: " + str + " For fieldType :" + fieldType);
+ return null;
}
}
@@ -436,4 +472,25 @@ public class QueryGeneration extends QueryGenerationBase {
}
return originalKey;
}
+
+ public boolean checkTokenizer(String fieldType,Class tokenizerFactoryClass,SolrDaoBase solrDaoBase) {
+ HashMap<String, Object> fieldTypeMap = solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase);
+ HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap
+ .get("analyzer");
+ if (analyzer != null) {
+ HashMap<String, Object> tokenizerMap = (HashMap<String, Object>) analyzer
+ .get("tokenizer");
+ if (tokenizerMap != null) {
+ String tokenizerClass = (String) tokenizerMap.get("class");
+ if (!StringUtils.isEmpty(tokenizerClass)) {
+ tokenizerClass =tokenizerClass.replace("solr.", "");
+ if (tokenizerClass.equalsIgnoreCase(tokenizerFactoryClass
+ .getSimpleName())) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
index a49107d..a128098 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
@@ -24,6 +24,8 @@ import java.util.List;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.ambari.logsearch.util.QueryBase;
import org.apache.ambari.logsearch.util.SolrUtil;
@@ -34,6 +36,7 @@ import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.gson.Gson;
+import com.sun.jersey.api.spring.Autowire;
public abstract class QueryGenerationBase extends QueryBase {
@@ -47,6 +50,14 @@ public abstract class QueryGenerationBase extends QueryBase {
@Autowired
JSONUtil jsonUtil;
+
+ @Autowired
+ AuditSolrDao auditSolrDao;
+
+ @Autowired
+ ServiceLogsSolrDao serviceLogsSolrDao;
+
+
public static enum CONDITION {
OR, AND
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
index bdd304f..2661150 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
@@ -23,9 +23,9 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.ambari.logsearch.manager.MgrBase;
import org.apache.log4j.Logger;
import org.codehaus.jettison.json.JSONArray;
@@ -38,7 +38,6 @@ public class ConfigUtil {
public static HashMap<String, String> auditLogsColumnMapping = new HashMap<String, String>();
- public static HashMap<String, String> schemaFieldsName = new HashMap<String, String>();
public static void initializeApplicationConfig() {
initializeColumnMapping();
@@ -78,57 +77,63 @@ public class ConfigUtil {
auditLogsColumnMapping);
}
+
public static void extractSchemaFieldsName(String responseString,
- String suffix) {
+ HashMap<String, String> schemaFieldsNameMap,
+ HashMap<String, String> schemaFieldTypeMap) {
try {
JSONObject jsonObject = new JSONObject(responseString);
- JSONArray jsonArrayList = jsonObject.getJSONArray("fields");
-
- if(jsonArrayList == null){
+ JSONObject schemajsonObject = jsonObject.getJSONObject("schema");
+ JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields");
+ JSONArray fieldTypeJsonArray = schemajsonObject
+ .getJSONArray("fieldTypes");
+ if (jsonArrayList == null) {
+ return;
+ }
+ if (fieldTypeJsonArray == null) {
return;
}
+ HashMap<String, String> _schemaFieldTypeMap = new HashMap<String, String>();
+ HashMap<String, String> _schemaFieldsNameMap = new HashMap<String, String>();
+ for (int i = 0; i < fieldTypeJsonArray.length(); i++) {
+ JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i);
+ String name = typeObject.getString("name");
+ String fieldTypeJson = typeObject.toString();
+ _schemaFieldTypeMap.put(name, fieldTypeJson);
+ }
for (int i = 0; i < jsonArrayList.length(); i++) {
JSONObject explrObject = jsonArrayList.getJSONObject(i);
String name = explrObject.getString("name");
String type = explrObject.getString("type");
-
if (!name.contains("@") && !name.startsWith("_")
- && !name.contains("_md5") && !name.contains("_ms")
- && !name.contains(LogSearchConstants.NGRAM_SUFFIX)
- && !name.contains("tags") && !name.contains("_str")) {
- schemaFieldsName.put(name + suffix, type);
+ && !name.contains("_md5") && !name.contains("_ms")
+ && !name.contains(LogSearchConstants.NGRAM_SUFFIX)
+ && !name.contains("tags") && !name.contains("_str")) {
+ _schemaFieldsNameMap.put(name, type);
}
}
-
+ schemaFieldsNameMap.clear();
+ schemaFieldTypeMap.clear();
+ schemaFieldsNameMap.putAll(_schemaFieldsNameMap);
+ schemaFieldTypeMap.putAll(_schemaFieldTypeMap);
} catch (Exception e) {
-
logger.error(e + "Credentials not specified in logsearch.properties "
- + MessageEnums.ERROR_SYSTEM);
-
+ + MessageEnums.ERROR_SYSTEM);
}
-
}
@SuppressWarnings("rawtypes")
- public static void getSchemaFieldsName(String suffix, String excludeArray[],
- List<String> fieldNames) {
- if (!schemaFieldsName.isEmpty()) {
- Iterator iteratorSechmaFieldsName = schemaFieldsName.entrySet()
- .iterator();
-
+ public static void getSchemaFieldsName(String excludeArray[],
+ List<String> fieldNames, SolrDaoBase solrDaoBase) {
+ if (!solrDaoBase.schemaFieldsNameMap.isEmpty()) {
+ Iterator iteratorSechmaFieldsName = solrDaoBase.schemaFieldsNameMap
+ .entrySet().iterator();
while (iteratorSechmaFieldsName.hasNext()) {
-
- Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName
- .next();
+ Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName.next();
String field = "" + fieldName.getKey();
-
- if (field.contains(suffix)) {
- field = field.replace(suffix, "");
- if (!isExclude(field, excludeArray)) {
- fieldNames.add(field);
- }
-
+ if (!isExclude(field, excludeArray)) {
+ fieldNames.add(field);
}
}
}
@@ -144,5 +149,4 @@ public class ConfigUtil {
}
return false;
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11984a06/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
index f68891f..ee706bf 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
@@ -20,10 +20,16 @@
package org.apache.ambari.logsearch.util;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Locale;
import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.log4j.Logger;
+import org.apache.solr.schema.TrieDoubleField;
+import org.apache.solr.schema.TrieFloatField;
+import org.apache.solr.schema.TrieIntField;
+import org.apache.solr.schema.TrieLongField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -34,6 +40,9 @@ public class SolrUtil {
@Autowired
StringUtil stringUtil;
+ @Autowired
+ JSONUtil jsonUtil;
+
public String setField(String fieldName, String value) {
if (value == null || value.trim().length() == 0) {
return "";
@@ -296,4 +305,49 @@ public class SolrUtil {
return newSearch.replace(" ", "\\ ");
}
+
+
+ public boolean isSolrFieldNumber(String fieldType,SolrDaoBase solrDaoBase) {
+ if (stringUtil.isEmpty(fieldType)) {
+ return false;
+ } else {
+ HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType,solrDaoBase);
+ if (typeInfoMap == null || typeInfoMap.isEmpty()) {
+ return false;
+ }
+ String fieldTypeClassName = (String) typeInfoMap.get("class");
+ if (fieldTypeClassName.equalsIgnoreCase(TrieIntField.class
+ .getSimpleName())) {
+ return true;
+ }
+ if (fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class
+ .getSimpleName())) {
+ return true;
+ }
+ if (fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class
+ .getSimpleName())) {
+ return true;
+ }
+ if (fieldTypeClassName.equalsIgnoreCase(TrieLongField.class
+ .getSimpleName())) {
+ return true;
+ }
+ return false;
+ }
+ }
+
+ public HashMap<String, Object> getFieldTypeInfoMap(String fieldType,SolrDaoBase solrDaoBase) {
+ String fieldTypeMetaData = solrDaoBase.schemaFieldTypeMap.get(fieldType);
+ HashMap<String, Object> fieldTypeMap = jsonUtil
+ .jsonToMapObject(fieldTypeMetaData);
+ if (fieldTypeMap == null) {
+ return new HashMap<String, Object>();
+ }
+ String classname = (String) fieldTypeMap.get("class");
+ if (!stringUtil.isEmpty(classname)) {
+ classname = classname.replace("solr.", "");
+ fieldTypeMap.put("class", classname);
+ }
+ return fieldTypeMap;
+ }
}
\ No newline at end of file