You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/04/25 17:14:44 UTC
[9/9] ambari git commit: AMBARI-16034. Incremental changes to
LogSearch to bring it up to date in the trunk (Dharmesh Makwana via oleewere)
AMBARI-16034. Incremental changes to LogSearch to bring it up to date in the trunk (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/888faf26
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/888faf26
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/888faf26
Branch: refs/heads/trunk
Commit: 888faf260d2d9fc454072ed69d1149c733ca98f0
Parents: 25ed583
Author: oleewere <ol...@gmail.com>
Authored: Mon Apr 25 16:48:31 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Apr 25 17:10:34 2016 +0200
----------------------------------------------------------------------
.../logsearch/appender/LogsearchAppender.java | 50 -
.../logsearch/appender/LogsearchConversion.java | 6 +
.../appender/LogsearchRollingFileAppender.java | 43 +
.../src/test/resources/log4j.properties | 11 +-
.../ambari-logsearch-logfeeder/build.xml | 3 -
.../ambari-logsearch-logfeeder/pom.xml | 8 +-
.../apache/ambari/logfeeder/LogFeederUtil.java | 11 +-
.../logconfig/FetchConfigFromSolr.java | 52 +-
.../logfeeder/logconfig/LogfeederScheduler.java | 3 +-
.../logconfig/filter/ApplyLogFilter.java | 7 +-
.../logconfig/filter/DefaultDataFilter.java | 5 +-
.../logconfig/filter/FilterLogData.java | 2 +-
.../apache/ambari/logfeeder/output/Output.java | 14 +
.../ambari/logfeeder/output/OutputKafka.java | 2 +-
.../ambari/logfeeder/output/OutputSolr.java | 3 +-
.../apache/ambari/logfeeder/util/SolrUtil.java | 28 +-
.../src/main/resources/config.json.j2 | 21 +-
.../src/main/resources/log4j.xml | 17 +-
.../src/main/scripts/run.sh | 6 +
.../logfeeder/filter/JSONFilterCodeTest.java | 10 +-
.../ambari-logsearch-portal/build.xml | 3 -
.../ambari-logsearch-portal/pom.xml | 7 +-
ambari-logsearch/ambari-logsearch-portal/run.sh | 8 +-
.../configsets/audit_logs/conf/managed-schema | 9 +-
.../configsets/hadoop_logs/conf/managed-schema | 37 +-
.../main/configsets/history/conf/managed-schema | 98 +
.../src/main/configsets/history/conf/schema.xml | 49 -
.../main/configsets/history/conf/solrconfig.xml | 1885 +++++++++++++++++-
.../logsearch/common/LogSearchConstants.java | 25 +
.../logsearch/common/ManageStartEndTime.java | 6 +-
.../ambari/logsearch/common/MessageEnums.java | 33 +-
.../ambari/logsearch/common/SearchCriteria.java | 9 +-
.../logsearch/common/UserSessionInfo.java | 2 +-
.../ambari/logsearch/dao/AuditSolrDao.java | 5 +
.../logsearch/dao/ServiceLogsSolrDao.java | 6 +
.../ambari/logsearch/dao/SolrDaoBase.java | 113 +-
.../ambari/logsearch/dao/UserConfigSolrDao.java | 5 +
.../logsearch/graph/GraphDataGenerator.java | 414 ++++
.../logsearch/graph/GraphDataGeneratorBase.java | 316 ++-
.../logsearch/graph/GraphDataGnerator.java | 397 ----
.../ambari/logsearch/manager/AuditMgr.java | 302 ++-
.../ambari/logsearch/manager/LogFileMgr.java | 50 +-
.../ambari/logsearch/manager/LogsMgr.java | 1460 +++++++-------
.../ambari/logsearch/manager/MgrBase.java | 104 +-
.../ambari/logsearch/manager/PublicMgr.java | 11 +-
.../ambari/logsearch/manager/UserConfigMgr.java | 212 +-
.../ambari/logsearch/query/QueryGeneration.java | 465 +++--
.../logsearch/query/QueryGenerationBase.java | 338 ++--
.../apache/ambari/logsearch/rest/AuditREST.java | 2 -
.../ambari/logsearch/rest/DashboardREST.java | 22 +-
.../ambari/logsearch/rest/UserConfigREST.java | 2 -
.../apache/ambari/logsearch/util/BizUtil.java | 292 ++-
.../ambari/logsearch/util/ConfigUtil.java | 29 +-
.../apache/ambari/logsearch/util/DateUtil.java | 54 +-
.../apache/ambari/logsearch/util/FileUtil.java | 42 +-
.../apache/ambari/logsearch/util/JSONUtil.java | 5 +-
.../apache/ambari/logsearch/util/QueryBase.java | 61 +-
.../ambari/logsearch/util/RESTErrorUtil.java | 4 +-
.../apache/ambari/logsearch/util/SolrUtil.java | 177 +-
.../ambari/logsearch/util/StringUtil.java | 6 +-
.../logsearch/util/XMLPropertiesUtil.java | 57 +-
.../ambari/logsearch/view/VBarDataList.java | 6 +-
.../ambari/logsearch/view/VCountList.java | 3 +-
.../ambari/logsearch/view/VGroupList.java | 3 +-
.../org/apache/ambari/logsearch/view/VHost.java | 4 +-
.../ambari/logsearch/view/VNameValue.java | 5 +-
.../ambari/logsearch/view/VNameValueList.java | 2 +-
.../apache/ambari/logsearch/view/VNodeList.java | 2 +-
.../ambari/logsearch/view/VSolrLogList.java | 6 +-
.../apache/ambari/logsearch/view/VSummary.java | 16 +-
.../ambari/logsearch/view/VSummaryCount.java | 8 +-
.../logsearch/view/VSummaryCountList.java | 2 +-
.../ambari/logsearch/view/VUserConfig.java | 8 +-
.../ambari/logsearch/view/VUserConfigList.java | 8 +-
.../LogsearchLogoutSuccessHandler.java | 2 +-
.../web/listeners/SpringEventListener.java | 2 +-
...LogsearchAbstractAuthenticationProvider.java | 4 +-
.../LogsearchAuthenticationProvider.java | 6 +-
.../LogsearchFileAuthenticationProvider.java | 4 +
.../LogsearchLdapAuthenticationProvider.java | 4 +
.../LogsearchSimpleAuthenticationProvider.java | 4 +
.../src/main/resources/default.properties | 1 +
.../src/main/resources/log4j.xml | 47 +-
.../src/main/resources/logsearch.properties | 14 +-
.../src/main/resources/logsearch.properties.j2 | 5 +
.../src/main/scripts/run.sh | 5 +
.../src/main/webapp/index.html | 6 +-
.../src/main/webapp/scripts/utils/Utils.js | 72 +-
.../src/main/webapp/scripts/utils/ViewUtils.js | 7 +-
.../main/webapp/scripts/views/common/Header.js | 7 +-
.../webapp/scripts/views/common/TableLayout.js | 2 +-
.../dashboard/BubbleGraphTableLayoutView.js | 29 +-
.../scripts/views/dashboard/HostListView.js | 5 +-
.../views/dialog/SaveSearchFilterView.js | 4 +-
.../views/filter/CreateLogfeederFilterView.js | 113 +-
.../webapp/scripts/views/tabs/LogFileView.js | 13 +-
.../main/webapp/scripts/views/tabs/TreeView.js | 1 -
.../troubleshoot/TroubleShootLayoutView.js | 70 +-
.../src/main/webapp/styles/style.css | 120 +-
.../src/main/webapp/styles/style_v2.css | 2 +-
.../webapp/templates/common/Header_tmpl.html | 13 +-
.../dashboard/MainLayoutView_tmpl.html | 4 +-
.../filter/CreateLogfeederFilter_tmpl.html | 19 +-
.../webapp/templates/tabs/LogFileView_tmpl.html | 4 +-
.../TroubleShootLayoutView_tmpl.html | 3 +-
ambari-logsearch/pom.xml | 1 +
106 files changed, 5384 insertions(+), 2706 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java
deleted file mode 100644
index 4339a21..0000000
--- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logsearch.appender;
-
-import org.apache.log4j.DailyRollingFileAppender;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Logger;
-import org.apache.log4j.spi.LoggingEvent;
-
-public class LogsearchAppender extends DailyRollingFileAppender {
- private static Logger logger = Logger.getLogger(LogsearchAppender.class);
-
- public LogsearchAppender() {
- logger.debug("Initializing LogsearchAppender........... ");
- }
-
- @Override
- public void append(LoggingEvent event) {
- super.append(event);
- }
-
- @Override
- public void setLayout(Layout layout) {
- super.setLayout(layout);
- }
-
- protected void subAppend(LoggingEvent event) {
- this.qw.write(this.layout.format(event));
- if (shouldFlush(event)) {
- this.qw.flush();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
index b9e7527..dbdfe6c 100644
--- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
+++ b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
@@ -70,4 +70,10 @@ public class LogsearchConversion extends EnhancedPatternLayout {
e.printStackTrace(pw);
return sw.toString();
}
+
+ @Override
+ public boolean ignoresThrowable() {
+ //set false to ignore exception stacktrace
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java
new file mode 100644
index 0000000..6a93db5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.appender;
+
+import org.apache.log4j.Layout;
+import org.apache.log4j.Logger;
+import org.apache.log4j.RollingFileAppender;
+import org.apache.log4j.spi.LoggingEvent;
+
+public class LogsearchRollingFileAppender extends RollingFileAppender {
+ private static Logger logger = Logger.getLogger(LogsearchRollingFileAppender.class);
+
+ public LogsearchRollingFileAppender() {
+ logger.trace("Initializing LogsearchRollingFileAppender........... ");
+ }
+
+ @Override
+ public void append(LoggingEvent event) {
+ super.append(event);
+ }
+
+ @Override
+ public void setLayout(Layout layout) {
+ super.setLayout(layout);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties b/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties
index b51e87d..78fb66b 100644
--- a/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties
+++ b/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties
@@ -13,11 +13,12 @@
# log4j configuration used during build and unit tests
# Root logger option
-log4j.rootLogger=ALL, logsearchJson
+log4j.rootLogger=INFO, logsearchJson
# Redirect log messages to a logsearch json
-log4j.appender.logsearchJson=org.apache.ambari.logsearch.appender.LogsearchAppender
-log4j.appender.logsearchJson.File=target/logsearch/log.json
-log4j.appender.logsearchJson.datePattern='.'yyyy-MM-dd
-log4j.appender.allLog.Append=true
+log4j.appender.logsearchJson=org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender
+log4j.appender.logsearchJson.File=target/jsonlog/log.json
+log4j.appender.logsearchJson.maxFileSize=10MB
+log4j.appender.logsearchJson.maxBackupIndex=10
+log4j.appender.logsearchJson.Append=true
log4j.appender.logsearchJson.layout=org.apache.ambari.logsearch.appender.LogsearchConversion
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
index 53f893e..738b2ef 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
@@ -32,9 +32,6 @@
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset dir="target/libs"/>
</copy>
- <copy todir="target/package/" includeEmptyDirs="no">
- <fileset file="target/LogFeeder.jar"/>
- </copy>
<copy todir="target/package/classes" includeEmptyDirs="no">
<fileset dir="target/classes"/>
</copy>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index 0888010..dc1b361 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -39,6 +39,11 @@
<dependencies>
<dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-appender</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
@@ -79,6 +84,7 @@
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
+
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -109,8 +115,8 @@
<artifactId>jackson-xc</artifactId>
<version>1.9.13</version>
</dependency>
- </dependencies>
+ </dependencies>
<build>
<finalName>LogFeeder</finalName>
<pluginManagement>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java
index 7303694..7a30d72 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java
@@ -33,9 +33,11 @@ import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.TimeZone;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
import org.apache.ambari.logfeeder.mapper.Mapper;
import org.apache.ambari.logfeeder.output.Output;
import org.apache.commons.lang3.StringUtils;
@@ -58,6 +60,7 @@ public class LogFeederUtil {
final static int HASH_SEED = 31174077;
public final static String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";
+ public final static String SOLR_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
static Gson gson = new GsonBuilder().setDateFormat(DATE_FORMAT).create();
static Properties props;
@@ -324,6 +327,9 @@ public class LogFeederUtil {
}
public static Map<String, Object> toJSONObject(String jsonStr) {
+ if(jsonStr==null || jsonStr.trim().isEmpty()){
+ return new HashMap<String, Object>();
+ }
Type type = new TypeToken<Map<String, Object>>() {
}.getType();
return gson.fromJson(jsonStr, type);
@@ -380,7 +386,8 @@ public class LogFeederUtil {
public static String getDate(String timeStampStr) {
try {
- DateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
+ DateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT);
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
Date netDate = (new Date(Long.parseLong(timeStampStr)));
return sdf.format(netDate);
} catch (Exception ex) {
@@ -468,7 +475,7 @@ public class LogFeederUtil {
return true;
}
}
- if (value.equalsIgnoreCase("ALL")) {
+ if (value.equalsIgnoreCase(LogFeederConstants.ALL)) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
index fc12458..e645a3d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
@@ -40,12 +40,18 @@ public class FetchConfigFromSolr extends Thread {
private static String endTimeDateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSS";//2016-04-05T04:30:00.000Z
private static String sysTimeZone = "GMT";
- public FetchConfigFromSolr() {
+ public FetchConfigFromSolr(boolean isDaemon) {
this.setName(this.getClass().getSimpleName());
+ this.setDaemon(isDaemon);
}
@Override
public void run() {
+ String zkHosts = LogFeederUtil.getStringProperty("logfeeder.solr.zkhosts");
+ if( zkHosts == null || zkHosts.trim().length() == 0 ) {
+ logger.warn("Solr ZKHosts for UserConfig/History is not set. Won't look for level configuration from Solr.");
+ return;
+ }
solrConfigInterval = LogFeederUtil.getIntProperty("logfeeder.solr.config.internal", solrConfigInterval);
delay = 1000 * solrConfigInterval;
do {
@@ -60,11 +66,14 @@ public class FetchConfigFromSolr extends Thread {
}
private synchronized void pullConfigFromSolr() {
- HashMap<String, Object> configDocMap = SolrUtil.getInstance().getConfigDoc();
- if (configDocMap != null) {
- String configJson = (String) configDocMap.get(LogFeederConstants.VALUES);
- if (configJson != null) {
- logfeederFilterWrapper = LogFeederUtil.getGson().fromJson(configJson, VLogfeederFilterWrapper.class);
+ SolrUtil solrUtil = SolrUtil.getInstance();
+ if(solrUtil!=null){
+ HashMap<String, Object> configDocMap = solrUtil.getConfigDoc();
+ if (configDocMap != null) {
+ String configJson = (String) configDocMap.get(LogFeederConstants.VALUES);
+ if (configJson != null) {
+ logfeederFilterWrapper = LogFeederUtil.getGson().fromJson(configJson, VLogfeederFilterWrapper.class);
+ }
}
}
}
@@ -117,19 +126,28 @@ public class FetchConfigFromSolr extends Thread {
List<String> hosts = componentFilter.getHosts();
List<String> defaultLevels = componentFilter.getDefaultLevels();
List<String> overrideLevels = componentFilter.getOverrideLevels();
- if (LogFeederUtil.isListContains(hosts, hostName, false)) {
- if (isFilterExpired(componentFilter)) {
- // pick default
- logger.debug("Filter for component " + componentName + " and host :" + hostName + " is expired at "
- + componentFilter.getExpiryTime());
- return defaultLevels;
- } else {
- // return tmp filter levels
- return overrideLevels;
+ String expiryTime=componentFilter.getExpiryTime();
+ //check is user override or not
+ if ((expiryTime != null && !expiryTime.isEmpty())
+ || (overrideLevels != null && !overrideLevels.isEmpty())
+ || (hosts != null && !hosts.isEmpty())) {
+ if (hosts == null || hosts.isEmpty()) {
+ // hosts list is empty or null consider it apply on all hosts
+ hosts.add(LogFeederConstants.ALL);
+ }
+ if (LogFeederUtil.isListContains(hosts, hostName, false)) {
+ if (isFilterExpired(componentFilter)) {
+ // pick default
+ logger.debug("Filter for component " + componentName + " and host :"
+ + hostName + " is expired at " + componentFilter.getExpiryTime());
+ return defaultLevels;
+ } else {
+ // return tmp filter levels
+ return overrideLevels;
+ }
}
- } else {
- return defaultLevels;
}
+ return defaultLevels;
}
public static VLogfeederFilter findComponentFilter(String componentName) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
index 7525dff..128c5c4 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
@@ -52,7 +52,8 @@ public enum LogfeederScheduler {
private List<Thread> getThreadList() {
List<Thread> tasks = new ArrayList<Thread>();
- tasks.add(new FetchConfigFromSolr());
+ Thread configMonitor = new FetchConfigFromSolr(true);
+ tasks.add(configMonitor);
return tasks;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
index 3748445..f223207 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
@@ -50,7 +50,12 @@ public class ApplyLogFilter extends DefaultDataFilter {
//return default value if there is no filter found for particular component
return defaultValue;
}
- List<String> allowedLevels = FetchConfigFromSolr.getAllowedLevels(hostName, componentFilter);
+ List<String> allowedLevels = FetchConfigFromSolr.getAllowedLevels(
+ hostName, componentFilter);
+ if (allowedLevels == null || allowedLevels.isEmpty()) {
+ // if allowedlevels list is empty then allow everything
+ allowedLevels.add(LogFeederConstants.ALL);
+ }
return LogFeederUtil.isListContains(allowedLevels, level, false);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
index 9e98c6a..a064663 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.logfeeder.logconfig.filter;
-import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
@@ -28,9 +27,7 @@ import org.apache.log4j.Logger;
*/
public class DefaultDataFilter {
private static Logger logger = Logger.getLogger(DefaultDataFilter.class);
-
- protected static final boolean CASE_SENSITIVE = false;
-
+
public boolean applyFilter(Map<String, Object> outputJsonObj, boolean defaultValue) {
return defaultValue;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
index 643df98..bf33f93 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
@@ -40,7 +40,7 @@ public enum FilterLogData {
return defaultValue;
}
Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
- return applyLogFilter.applyFilter(jsonObj, defaultValue);
+ return isAllowed(jsonObj);
}
public boolean isAllowed(Map<String, Object> jsonObj) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
index dd67d07..c067680 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
@@ -22,6 +22,7 @@ package org.apache.ambari.logfeeder.output;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.ambari.logfeeder.ConfigBlock;
import org.apache.ambari.logfeeder.LogFeederUtil;
@@ -115,5 +116,18 @@ public abstract class Output extends ConfigBlock {
logStatForMetric(writeBytesMetric, "Stat: Bytes Written");
}
+
+ public void trimStrValue(Map<String, Object> jsonObj) {
+ if (jsonObj != null) {
+ for (Entry<String, Object> entry : jsonObj.entrySet()) {
+ String key = entry.getKey();
+ Object value = entry.getValue();
+ if (value != null && value instanceof String) {
+ String valueStr = value.toString().trim();
+ jsonObj.put(key, valueStr);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
index cd4f951..efbc366 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
@@ -283,4 +283,4 @@ public class OutputKafka extends Output {
}
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index 215f691..6fb0b0e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -194,6 +194,7 @@ public class OutputSolr extends Output {
@Override
public void write(Map<String, Object> jsonObj, InputMarker inputMarker) throws Exception {
try {
+ trimStrValue(jsonObj);
outgoingBuffer.put(new OutputData(jsonObj, inputMarker));
} catch (InterruptedException e) {
// ignore
@@ -442,4 +443,4 @@ public class OutputSolr extends Output {
return localBuffer.isEmpty();
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
index 4265dc6..200a603 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import org.apache.ambari.logfeeder.LogFeederUtil;
import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
+import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
@@ -64,7 +65,12 @@ public class SolrUtil {
try {
instance = new SolrUtil();
} catch (Exception e) {
- logger.error(e);
+ final String LOG_MESSAGE_KEY = SolrUtil.class
+ .getSimpleName() + "_SOLR_UTIL";
+ LogFeederUtil.logErrorMessageByInterval(
+ LOG_MESSAGE_KEY,
+ "Error constructing solrUtil", e, logger,
+ Level.WARN);
}
}
}
@@ -162,6 +168,7 @@ public class SolrUtil {
QueryResponse queryResponse = solrClient.query(solrQuery, METHOD.POST);
return queryResponse;
} else {
+ logger.error("solrClient can't be null");
return null;
}
}
@@ -176,15 +183,18 @@ public class SolrUtil {
String fq = LogFeederConstants.ROW_TYPE + ":" + LogFeederConstants.NAME;
solrQuery.setFilterQueries(fq);
try {
- QueryResponse response = SolrUtil.getInstance().process(solrQuery);
- SolrDocumentList documentList = response.getResults();
- if (documentList != null && documentList.size() > 0) {
- SolrDocument configDoc = documentList.get(0);
- String configJson = LogFeederUtil.getGson().toJson(configDoc);
- configMap = (HashMap<String, Object>) LogFeederUtil.toJSONObject(configJson);
+ QueryResponse response = process(solrQuery);
+ if (response != null) {
+ SolrDocumentList documentList = response.getResults();
+ if (documentList != null && documentList.size() > 0) {
+ SolrDocument configDoc = documentList.get(0);
+ String configJson = LogFeederUtil.getGson().toJson(configDoc);
+ configMap = (HashMap<String, Object>) LogFeederUtil
+ .toJSONObject(configJson);
+ }
}
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error(e);
+ } catch (Exception e) {
+ logger.error("Error getting config", e);
}
return configMap;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
index 163ee2b..1c5ee8d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2
@@ -179,17 +179,17 @@
{
"type":"logsearch_app",
"rowtype":"service",
- "path":"{{logsearch_log_dir}}/logsearch.log"
+ "path":"{{logsearch_log_dir}}/logsearch.json"
},
{
"type":"logsearch_feeder",
"rowtype":"service",
- "path":"{{logfeeder_log_dir}}/logfeeder.log"
+ "path":"{{logfeeder_log_dir}}/logfeeder.json"
},
{
"type":"logsearch_perf",
"rowtype":"service",
- "path":"{{logsearch_log_dir}}/logsearch-performance.log"
+ "path":"{{logsearch_log_dir}}/logsearch-performance.json"
},
{
"type":"ranger_admin",
@@ -625,13 +625,22 @@
},
{
+ "filter": "json",
+ "conditions": {
+ "fields": {
+ "type": [
+ "logsearch_app",
+ "logsearch_feeder",
+ "logsearch_perf"
+ ]
+ }
+ }
+ },
+ {
"filter":"grok",
"conditions":{
"fields":{
"type":[
- "logsearch_app",
- "logsearch_feeder",
- "logsearch_perf",
"ranger_admin",
"ranger_dbpatch"
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
index 0717477..7ef967c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
@@ -33,24 +33,35 @@
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/>
</layout>
+ </appender>
+
+ <appender name="rolling_file_json"
+ class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
+ <param name="file" value="logs/logsearch-logfeeder.json" />
+ <param name="append" value="true" />
+ <param name="maxFileSize" value="10MB" />
+ <param name="maxBackupIndex" value="10" />
+ <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" />
</appender>
+
<!-- Logs to suppress BEGIN -->
<category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
<priority value="error" />
- <appender-ref ref="rolling_file" />
+ <appender-ref ref="daily_rolling_file" />
</category>
<category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false">
<priority value="fatal" />
- <appender-ref ref="rolling_file" />
+ <appender-ref ref="daily_rolling_file" />
</category>
<!-- Logs to suppress END -->
<category name="org.apache.ambari.logfeeder" additivity="false">
- <priority value="info" />
+ <priority value="INFO" />
<appender-ref ref="console" />
<!-- <appender-ref ref="daily_rolling_file" /> -->
+ <appender-ref ref="rolling_file_json"/>
</category>
<root>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
index 1e7185a..356a358 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
@@ -47,6 +47,12 @@ fi
if [ "$LOGFEEDER_CONF_DIR" = "" ]; then
LOGFEEDER_CONF_DIR="/etc/logfeeder/conf"
+ if [ ! -d $LOGFEEDER_CONF_DIR ]; then
+ if [ -d $script_dir/classes ]; then
+ LOGFEEDER_CONF_DIR=$script_dir/classes
+ fi
+ fi
+
fi
LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java
index ebfd0f5..fd14b48 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java
@@ -18,10 +18,12 @@
package org.apache.ambari.logfeeder.filter;
+import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import java.util.TimeZone;
import org.apache.ambari.logfeeder.LogFeederUtil;
import org.apache.ambari.logfeeder.OutputMgr;
@@ -64,7 +66,9 @@ public class JSONFilterCodeTest {
EasyMock.replay(mockOutputMgr);
Date d = new Date();
- String dateString = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).format(d);
+ DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT);
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String dateString = sdf.format(d);
jsonFilterCode.apply("{ logtime: '" + d.getTime() + "', line_number: 100 }", new InputMarker());
EasyMock.verify(mockOutputMgr);
@@ -86,7 +90,9 @@ public class JSONFilterCodeTest {
EasyMock.replay(mockOutputMgr);
Date d = new Date();
- String dateString = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).format(d);
+ DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT);
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String dateString = sdf.format(d);
jsonFilterCode.apply("{ logtime: '" + d.getTime() + "', some_field: 'abc' }", new InputMarker());
EasyMock.verify(mockOutputMgr);
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/build.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/build.xml b/ambari-logsearch/ambari-logsearch-portal/build.xml
index 22c7938..d6db5f0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/build.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/build.xml
@@ -38,9 +38,6 @@
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset dir="target/libs"/>
</copy>
- <copy todir="target/package/" includeEmptyDirs="no">
- <fileset file="target/LogSearch.jar"/>
- </copy>
<copy todir="target/package/classes" includeEmptyDirs="no">
<fileset dir="target/classes"/>
</copy>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 9c5ba72..799e95e 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -163,6 +163,7 @@
<exclude>src/main/webapp/styles/animate.css</exclude>
<exclude>**/*.json</exclude>
<exclude>src/main/webapp/styles/visualsearch.css</exclude>
+ <exclude>**/*.log</exclude>
</excludes>
</configuration>
<executions>
@@ -542,6 +543,10 @@
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-appender</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/run.sh b/ambari-logsearch/ambari-logsearch-portal/run.sh
index 4b1be8b..ae3c779 100755
--- a/ambari-logsearch/ambari-logsearch-portal/run.sh
+++ b/ambari-logsearch/ambari-logsearch-portal/run.sh
@@ -17,6 +17,8 @@ echo "
███████╗╚██████╔╝╚██████╔╝ ███████║███████╗██║ ██║██║ ██║╚██████╗██║ ██║
╚══════╝ ╚═════╝ ╚═════╝ ╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝
"
-mvn clean compile -Pdev
-mvn exec:java -Pdev
-
+cd ..
+mvn clean compile package -Pdev
+cd ambari-logsearch-portal
+#mvn exec:java -Pdev
+java -cp target/libs/*:target/classes/ org.apache.ambari.logsearch.LogSearch
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
index 4a6d3ac..d2a1b6d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
@@ -47,6 +47,7 @@
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="n_gram" class="solr.TextField" sortMissingLast="true" omitNorms="true">
@@ -103,17 +104,13 @@
<field name="req_self_id" type="key_lower_case" multiValued="false"/>
<field name="event_md5" type="string" multiValued="false"/>
<field name="file" type="key_lower_case" multiValued="false"/>
- <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/>
+ <field name="log_message" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
<field name="logfile_line_number" type="tint" omitNorms="false"/>
- <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/>
+ <!-- <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/> -->
<field name="message_md5" type="string" multiValued="false"/>
<field name="type" type="key_lower_case" multiValued="false"/>
<dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/>
<dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/>
- <!-- Copying everything text -->
- <copyField source="*" dest="text" maxChars="25000" />
-
-
</schema>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema
index fa33516..cad0955 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema
@@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
<!-- Solr managed schema - automatically generated - DO NOT EDIT -->
<schema name="hadoop-logs-schema" version="1.5">
<uniqueKey>id</uniqueKey>
@@ -51,6 +68,7 @@
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/>
@@ -78,12 +96,12 @@
<field name="ip" type="ip_address" multiValued="false"/>
<field name="level" type="key_lower_case" multiValued="false"/>
<field name="line_number" type="tint" omitNorms="false"/>
- <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/>
+ <field name="log_message" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
<field name="logfile_line_number" type="tint" omitNorms="false"/>
<field name="logger_name" type="key_lower_case" multiValued="false"/>
<field name="logtime" type="tdate" multiValued="false" docValues="true"/>
<field name="logtype" type="key_lower_case" multiValued="false"/>
- <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/>
+ <!-- <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/> -->
<field name="message_md5" type="string" multiValued="false"/>
<field name="method" type="key_lower_case" multiValued="false" omitNorms="false"/>
<field name="path" type="key_lower_case" multiValued="false"/>
@@ -96,11 +114,16 @@
<dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/>
<dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/>
-
+ <dynamicField name='key_*' type="key_lower_case" multiValued="false" stored="false"/>
+ <dynamicField name="ws_*" type="text_ws" multiValued="false" omitNorms="false" stored="false"/>
+ <copyField source="log_message" dest="key_log_message"/>
+ <!-- <copyField source="log_message" dest="ws_log_message"/> -->
+
<!-- Copy Fields-->
<!-- ngram fields -->
<!-- Whenever using a copy field provide following structure "ngram_"<OriginalFieldName> -->
+<!-- Commenting till we test properly. Now it is not used and it is taking unnecessary cpu, memory and disk space
<copyField source="bundle_id" dest="ngram_bundle_id"/>
<copyField source="case_id" dest="ngram_case_id"/>
<copyField source="cluster" dest="ngram_cluster"/>
@@ -115,11 +138,7 @@
<copyField source="thread_name" dest="ngram_thread_name"/>
<copyField source="type" dest="ngram_type"/>
- <!-- Standard text tokenizer -->
- <copyField source="log_message" dest="std_log_message"/>
-
- <!-- Copying everything text -->
- <copyField source="*" dest="text" maxChars="25000" />
-
+-->
+
</schema>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema
new file mode 100644
index 0000000..062c14f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Solr managed schema - automatically generated - DO NOT EDIT -->
+<schema name="history" version="1.1">
+
+ <uniqueKey>composite_filtername-username</uniqueKey>
+
+ <fieldType name="binary" class="solr.BinaryField"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
+ <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/>
+ <fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="ip_address" class="solr.TextField">
+ <analyzer>
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" replace="." delimiter="."/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/>
+ <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="n_gram" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <tokenizer class="solr.NGramTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="key_lower_case" class="solr.TextField"
+ sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+
+ <solrQueryParser defaultOperator="OR"/>
+
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="composite_filtername-username" type="string"/>
+ <field name="filtername" type="key_lower_case" indexed="true" required="true" stored="true"/>
+ <field name="id" type="string" required="true"/>
+ <field name="jsons" type="string"/>
+ <field name="rowtype" type="string" required="true"/>
+ <field name="share_username_list" type="string" multiValued="true"/>
+ <field name="username" type="key_lower_case" indexed="true" required="true" stored="true"/>
+
+ <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/>
+ <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/>
+ <dynamicField name="*_string" type="string" multiValued="true" indexed="true" stored="true"/>
+
+
+</schema>
http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml
deleted file mode 100644
index 2b9515c..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version='1.0' ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<schema name='history' version='1.1'>
- <types>
- <fieldtype name='string' class='solr.StrField' />
- <fieldtype name='long' class='solr.TrieLongField' />
-
- <fieldType name="key_lower_case" class="solr.TextField"
- sortMissingLast="true" omitNorms="true">
- <analyzer>
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory" />
- </analyzer>
- </fieldType>
- </types>
-
- <fields>
- <field name='id' type='string' required='true' />
- <field name="_version_" type="long" indexed="true" stored="true"/>
-
- <field name='username' type='key_lower_case' indexed="true" stored="true" required='true'/>
- <field name='jsons' type='string'/>
-
- <field name="rowtype" type='string' required='true'/>
- <field name='filtername' type='key_lower_case' indexed="true" stored="true" required='true'/>
-
- <field name='share_username_list' type='string' multiValued='true' />
- <field name='composite_filtername-username' type='string'/>
-
- <dynamicField name='*_string' type='string' multiValued='true' indexed='true' stored='true'/>
- </fields>
- <uniqueKey>composite_filtername-username</uniqueKey>
- <solrQueryParser defaultOperator='OR' />
-</schema>