You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/07/19 12:39:47 UTC
[ambari] branch trunk updated: AMBARI-23076. Log Search configurable zookeeper usage for config API … (#1775)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new d531263 AMBARI-23076. Log Search configurable zookeeper usage for config API … (#1775)
d531263 is described below
commit d531263b3955014acefd11820e7aee15b78a3115
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Thu Jul 19 14:39:45 2018 +0200
AMBARI-23076. Log Search configurable zookeeper usage for config API … (#1775)
* AMBARI-23076. Log Search configurable zookeeper usage for config API (use local implementation by default)
* AMBARI-23076. Add ambari-server related changes.
* AMBARI-23076. Add properties to turn on local config + solr log level filter management.
---
...earchConfig.java => LogLevelFilterManager.java} | 43 +++---
.../config/api/LogLevelFilterMonitor.java | 8 +-
.../config/api/LogLevelFilterUpdater.java | 66 ++++++++
.../logsearch/config/api/LogSearchConfig.java | 14 +-
.../config/api/LogSearchConfigLogFeeder.java | 20 ---
.../config/api/LogSearchConfigServer.java | 27 ----
.../logsearch/config/api/OutputConfigMonitor.java | 9 --
.../api/model/outputconfig/OutputProperties.java | 23 ---
.../config/api/LogSearchConfigLogFeederClass1.java | 18 +--
.../config/api/LogSearchConfigLogFeederClass2.java | 19 +--
.../config/api/LogSearchConfigServerClass1.java | 27 ++--
.../config/api/LogSearchConfigServerClass2.java | 26 ++--
.../pom.xml | 32 +---
.../ambari/logsearch/config/json/JsonHelper.java | 60 +++++++
.../model/inputconfig/impl/ConditionsImpl.java | 2 +-
.../json}/model/inputconfig/impl/FieldsImpl.java | 2 +-
.../model/inputconfig/impl/FilterAdapter.java | 2 +-
.../inputconfig/impl/FilterDescriptorImpl.java | 2 +-
.../inputconfig/impl/FilterGrokDescriptorImpl.java | 2 +-
.../inputconfig/impl/FilterJsonDescriptorImpl.java | 2 +-
.../impl/FilterKeyValueDescriptorImpl.java | 2 +-
.../json}/model/inputconfig/impl/InputAdapter.java | 2 +-
.../model/inputconfig/impl/InputConfigGson.java | 4 +-
.../model/inputconfig/impl/InputConfigImpl.java | 2 +-
.../impl/InputCustomDescriptorImpl.java | 2 +-
.../inputconfig/impl/InputDescriptorImpl.java | 2 +-
.../impl/InputFileBaseDescriptorImpl.java | 2 +-
.../inputconfig/impl/InputFileDescriptorImpl.java | 2 +-
.../impl/InputS3FileDescriptorImpl.java | 2 +-
.../impl/MapAnonymizeDescriptorImpl.java | 2 +-
.../inputconfig/impl/MapCustomDescriptorImpl.java | 2 +-
.../inputconfig/impl/MapDateDescriptorImpl.java | 2 +-
.../impl/MapFieldCopyDescriptorImpl.java | 2 +-
.../inputconfig/impl/MapFieldDescriptorImpl.java | 2 +-
.../impl/MapFieldNameDescriptorImpl.java | 2 +-
.../impl/MapFieldValueDescriptorImpl.java | 2 +-
.../inputconfig/impl/PostMapValuesAdapter.java | 2 +-
.../model/inputconfig/impl/PostMapValuesImpl.java | 2 +-
.../ambari-logsearch-config-local/pom.xml | 44 ++++++
.../config/local/LogLevelFilterManagerLocal.java | 107 +++++++++++++
.../config/local/LogSearchConfigLocal.java | 87 +++++++++++
.../config/local/LogSearchConfigLocalUpdater.java | 161 +++++++++++++++++++
.../local/LogSearchConfigLogFeederLocal.java | 116 ++++++++++++++
.../config/local/LogSearchConfigServerLocal.java} | 64 +++++---
.../pom.xml | 54 ++-----
.../config/solr/LogLevelFilterManagerSolr.java | 172 +++++++++++++++++++++
.../config/solr/LogLevelFilterUpdaterSolr.java | 80 ++++++++++
.../ambari-logsearch-config-zookeeper/pom.xml | 22 +--
.../config/zookeeper/LogLevelFilterManagerZK.java | 94 +++++++++++
.../zookeeper/LogSearchConfigLogFeederZK.java | 60 +------
.../config/zookeeper/LogSearchConfigServerZK.java | 48 +-----
.../config/zookeeper/LogSearchConfigZK.java | 18 +--
.../ambari/logsearch/domain/StoryDataRegistry.java | 2 +-
.../ambari/logsearch/patterns/PatternITBase.java | 2 +-
.../logsearch/steps/LogSearchConfigApiSteps.java | 4 +-
.../pom.xml | 1 +
.../ambari/logfeeder/plugin/output/Output.java | 3 -
.../ambari-logsearch-logfeeder/pom.xml | 15 ++
.../ambari/logfeeder/common/ConfigHandler.java | 9 +-
.../logfeeder/common/LogEntryParseTester.java | 25 +--
.../logfeeder/common/LogFeederConstants.java | 11 ++
.../ambari/logfeeder/conf/ApplicationConfig.java | 58 ++++++-
.../ambari/logfeeder/conf/LogFeederProps.java | 73 ++++++++-
.../ambari/logfeeder/input/InputSimulate.java | 4 +-
.../loglevelfilter/LogLevelFilterHandler.java | 18 ++-
.../ambari/logfeeder/output/OutputDevNull.java | 5 -
.../apache/ambari/logfeeder/output/OutputFile.java | 5 -
.../ambari/logfeeder/output/OutputHDFSFile.java | 5 -
.../ambari/logfeeder/output/OutputKafka.java | 6 -
.../ambari/logfeeder/output/OutputS3File.java | 8 +-
.../apache/ambari/logfeeder/output/OutputSolr.java | 98 +++---------
.../shipper-conf/output.config-sample.json | 4 +-
.../src/main/resources/logfeeder.properties | 3 +
.../ambari/logfeeder/filter/FilterGrokTest.java | 2 +-
.../ambari/logfeeder/filter/FilterJSONTest.java | 2 +-
.../logfeeder/filter/FilterKeyValueTest.java | 2 +-
.../ambari/logfeeder/input/InputFileTest.java | 2 +-
.../logfeeder/mapper/MapperAnonymizeTest.java | 2 +-
.../ambari/logfeeder/mapper/MapperDateTest.java | 2 +-
.../logfeeder/mapper/MapperFieldCopyTest.java | 2 +-
.../logfeeder/mapper/MapperFieldNameTest.java | 2 +-
.../logfeeder/mapper/MapperFieldValueTest.java | 2 +-
.../logfeeder/output/OutputLineFilterTest.java | 2 +-
.../ambari/logfeeder/output/OutputManagerTest.java | 2 +-
ambari-logsearch/ambari-logsearch-server/pom.xml | 5 +
.../ambari/logsearch/common/MessageEnums.java | 1 +
.../common/ShipperConfigDescriptionStorage.java | 2 +-
.../ambari/logsearch/conf/ApplicationConfig.java | 1 -
.../logsearch/conf/LogSearchConfigApiConfig.java | 20 ++-
.../ambari/logsearch/conf/SecurityConfig.java | 27 +++-
.../ambari/logsearch/conf/SolrClientsHolder.java} | 39 ++---
.../apache/ambari/logsearch/conf/SolrConfig.java | 31 ++--
.../global/SolrLogLevelFilterManagerState.java} | 21 ++-
.../configurer/SolrCollectionConfigurer.java | 20 ++-
.../configurer/SolrLogLevelFilterConfigurer.java | 92 +++++++++++
.../apache/ambari/logsearch/dao/AuditSolrDao.java | 8 +-
.../ambari/logsearch/dao/EventHistorySolrDao.java | 8 +-
.../ambari/logsearch/dao/ServiceLogsSolrDao.java | 8 +-
.../logsearch/handler/CreateCollectionHandler.java | 13 +-
.../logsearch/manager/ShipperConfigManager.java | 20 ++-
.../LogSearchLogLevelFilterManagerFilter.java | 102 ++++++++++++
.../test-config/logfeeder/logfeeder.properties | 4 +-
.../logfeeder/shipper-conf/output.config.json | 2 +
.../test-config/logsearch/logsearch.properties | 3 +
ambari-logsearch/pom.xml | 3 +
.../configuration/logfeeder-output-config.xml | 2 +-
.../0.5.0/properties/output.config.json.j2 | 2 +
107 files changed, 1739 insertions(+), 645 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterManager.java
similarity index 64%
copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java
copy to ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterManager.java
index 6c3b910..00df7bf 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterManager.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,34 +16,37 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.ambari.logsearch.config.api;
-import java.io.Closeable;
-
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
-/**
- * Log Search Configuration, which uploads, retrieves configurations, and monitors it's changes.
- */
-public interface LogSearchConfig extends Closeable {
- /**
- * Uploads the input configuration for a service in a cluster.
- *
- * @param clusterName The name of the cluster where the service is.
- * @param serviceName The name of the service of which's input configuration is uploaded.
- * @param inputConfig The input configuration of the service.
- * @throws Exception
- */
- void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception;
+public interface LogLevelFilterManager {
/**
* Uploads the log level filter of a log.
- *
+ *
* @param clusterName The name of the cluster where the log is.
* @param logId The id of the log.
* @param filter The log level filter for the log.
- * @throws Exception
+ * @throws Exception
*/
void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) throws Exception;
+
+ /**
+ * Modifies the log level filters for all the logs.
+ *
+ * @param clusterName The name of the cluster where the logs are.
+ * @param filters The log level filters to set.
+ * @throws Exception
+ */
+ void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception;
+
+ /**
+ * Returns the Log Level Filters of a cluster.
+ *
+ * @param clusterName The name of the cluster which's log level filters are required.
+ * @return All the log level filters of the cluster.
+ */
+ LogLevelFilterMap getLogLevelFilters(String clusterName);
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterMonitor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterMonitor.java
index 766f751..841f09e 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterMonitor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterMonitor.java
@@ -24,8 +24,9 @@ package org.apache.ambari.logsearch.config.api;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-public interface LogLevelFilterMonitor {
+import java.util.Map;
+public interface LogLevelFilterMonitor {
/**
* Notification of a new or updated log level filter.
*
@@ -41,4 +42,9 @@ public interface LogLevelFilterMonitor {
*/
void removeLogLevelFilter(String logId);
+ /**
+ * Helper function to get all log level filters
+ */
+ Map<String, LogLevelFilter> getLogLevelFilters();
+
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterUpdater.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterUpdater.java
new file mode 100644
index 0000000..425cdc8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogLevelFilterUpdater.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.api;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used for connect a remote source periodically to get / set log level filters.
+ */
+public abstract class LogLevelFilterUpdater extends Thread {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterUpdater.class);
+
+ private final LogLevelFilterMonitor logLevelFilterMonitor;
+ private final int interval;
+ private boolean stop = false;
+
+ public LogLevelFilterUpdater(String threadName, LogLevelFilterMonitor logLevelFilterMonitor, Integer interval) {
+ this.setName(threadName);
+ this.setDaemon(true);
+ this.logLevelFilterMonitor = logLevelFilterMonitor;
+ this.interval = interval == null ? 30 : interval;
+ }
+
+ public LogLevelFilterMonitor getLogLevelFilterMonitor() {
+ return logLevelFilterMonitor;
+ }
+
+ public void setStop(boolean stop) {
+ this.stop = stop;
+ }
+
+ @Override
+ public void run() {
+ while (!Thread.currentThread().isInterrupted() || !stop) {
+ try {
+ Thread.sleep(1000 * interval);
+ checkFilters(logLevelFilterMonitor);
+ } catch (Exception e) {
+ LOG.error("Exception happened during log level filter check: {}", e);
+ }
+ }
+ }
+
+ /**
+ * Periodically check filters from a source (and use log level filter monitor to create/update/delete it)
+ */
+ protected abstract void checkFilters(final LogLevelFilterMonitor logLevelFilterMonitor);
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java
index 6c3b910..97eabdf 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java
@@ -38,12 +38,12 @@ public interface LogSearchConfig extends Closeable {
void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception;
/**
- * Uploads the log level filter of a log.
- *
- * @param clusterName The name of the cluster where the log is.
- * @param logId The id of the log.
- * @param filter The log level filter for the log.
- * @throws Exception
+ * Get log level filter handler / manager
+ */
+ LogLevelFilterManager getLogLevelFilterManager();
+
+ /**
+ * Set log level filter handler / manager
*/
- void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) throws Exception;
+ void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager);
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
index 1387515..af4408a 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
@@ -19,11 +19,8 @@
package org.apache.ambari.logsearch.config.api;
-import java.util.List;
import java.util.Map;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
-
/**
* Log Search Configuration for Log Feeder.
*/
@@ -56,21 +53,4 @@ public interface LogSearchConfigLogFeeder extends LogSearchConfig {
*/
void monitorInputConfigChanges(InputConfigMonitor inputConfigMonitor, LogLevelFilterMonitor logLevelFilterMonitor,
String clusterName) throws Exception;
-
- /**
- * Get the properties of an Output Solr.
- *
- * @param type The type of the Output Solr.
- * @return The properties of the Output Solr, or null if it doesn't exist.
- * @throws Exception
- */
- OutputSolrProperties getOutputSolrProperties(String type) throws Exception;
-
- /**
- * Saves the properties of an Output Solr.
- *
- * @param outputConfigMonitors The monitors which want to watch the output config changes.
- * @throws Exception
- */
- void monitorOutputProperties(List<? extends OutputConfigMonitor> outputConfigMonitors) throws Exception;
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServer.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServer.java
index 26889be..d269c5a 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServer.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServer.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
/**
@@ -82,30 +81,4 @@ public interface LogSearchConfigServer extends LogSearchConfig {
* @return The input configuration for the service if it exists, null otherwise.
*/
InputConfig getInputConfig(String clusterName, String serviceName);
-
- /**
- * Modifies the log level filters for all the logs.
- *
- * @param clusterName The name of the cluster where the logs are.
- * @param filters The log level filters to set.
- * @throws Exception
- */
- void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception;
-
- /**
- * Returns the Log Level Filters of a cluster.
- *
- * @param clusterName The name of the cluster which's log level filters are required.
- * @return All the log level filters of the cluster.
- */
- LogLevelFilterMap getLogLevelFilters(String clusterName);
-
- /**
- * Saves the properties of an Output Solr.
- *
- * @param type The type of the Output Solr.
- * @param outputSolrProperties The properties of the Output Solr.
- * @throws Exception
- */
- void saveOutputSolrProperties(String type, OutputSolrProperties outputSolrProperties) throws Exception;
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/OutputConfigMonitor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/OutputConfigMonitor.java
index c54626d..cad4a7d 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/OutputConfigMonitor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/OutputConfigMonitor.java
@@ -19,8 +19,6 @@
package org.apache.ambari.logsearch.config.api;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
-
/**
* Monitors output configuration changes.
*/
@@ -34,11 +32,4 @@ public interface OutputConfigMonitor {
* @return The type of the output logs.
*/
String getOutputType();
-
- /**
- * Will be called whenever there is a change in the configuration of the output.
- *
- * @param outputProperties The modified properties of the output.
- */
- void outputConfigChanged(OutputProperties outputProperties);
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/outputconfig/OutputProperties.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/outputconfig/OutputProperties.java
deleted file mode 100644
index affd5b9..0000000
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/outputconfig/OutputProperties.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logsearch.config.api.model.outputconfig;
-
-public interface OutputProperties {
-}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass1.java b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass1.java
index b7da7fe..54b3ca0 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass1.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass1.java
@@ -19,13 +19,8 @@
package org.apache.ambari.logsearch.config.api;
-import java.util.List;
import java.util.Map;
-import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
-
public class LogSearchConfigLogFeederClass1 implements LogSearchConfigLogFeeder {
@Override
public void init(Map<String, String> properties, String clusterName) {}
@@ -39,19 +34,18 @@ public class LogSearchConfigLogFeederClass1 implements LogSearchConfigLogFeeder
public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
@Override
- public void monitorInputConfigChanges(InputConfigMonitor inputConfigMonitor, LogLevelFilterMonitor logLevelFilterMonitor,
- String clusterName) throws Exception {}
+ public LogLevelFilterManager getLogLevelFilterManager() {
+ return null;
+ }
@Override
- public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) {}
+ public void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager) {
- @Override
- public OutputSolrProperties getOutputSolrProperties(String type) {
- return null;
}
@Override
- public void monitorOutputProperties(List<? extends OutputConfigMonitor> outputConfigMonitors) throws Exception {}
+ public void monitorInputConfigChanges(InputConfigMonitor inputConfigMonitor, LogLevelFilterMonitor logLevelFilterMonitor,
+ String clusterName) throws Exception {}
@Override
public void close() {}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass2.java b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass2.java
index b703bf8..ca23b22 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass2.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeederClass2.java
@@ -19,13 +19,8 @@
package org.apache.ambari.logsearch.config.api;
-import java.util.List;
import java.util.Map;
-import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
-
public class LogSearchConfigLogFeederClass2 implements LogSearchConfigLogFeeder {
@Override
public void init(Map<String, String> properties, String clusterName) {}
@@ -39,19 +34,17 @@ public class LogSearchConfigLogFeederClass2 implements LogSearchConfigLogFeeder
public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
@Override
- public void monitorInputConfigChanges(InputConfigMonitor inputConfigMonitor, LogLevelFilterMonitor logLevelFilterMonitor,
- String clusterName) throws Exception {}
-
- @Override
- public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) {}
+ public LogLevelFilterManager getLogLevelFilterManager() {
+ return null;
+ }
@Override
- public OutputSolrProperties getOutputSolrProperties(String type) {
- return null;
+ public void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager) {
}
@Override
- public void monitorOutputProperties(List<? extends OutputConfigMonitor> outputConfigMonitors) throws Exception {}
+ public void monitorInputConfigChanges(InputConfigMonitor inputConfigMonitor, LogLevelFilterMonitor logLevelFilterMonitor,
+ String clusterName) throws Exception {}
@Override
public void close() {}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java
index 71e924a..a1aaa25 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java
@@ -22,9 +22,6 @@ package org.apache.ambari.logsearch.config.api;
import java.util.List;
import java.util.Map;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
public class LogSearchConfigServerClass1 implements LogSearchConfigServer {
@@ -40,36 +37,32 @@ public class LogSearchConfigServerClass1 implements LogSearchConfigServer {
public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
@Override
- public void setInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
-
- @Override
- public List<String> getServices(String clusterName) {
+ public LogLevelFilterManager getLogLevelFilterManager() {
return null;
}
@Override
- public String getGlobalConfigs(String clusterName) {
- return null;
- }
+ public void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager) {
- @Override
- public InputConfig getInputConfig(String clusterName, String serviceName) {
- return null;
}
@Override
- public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) {}
+ public void setInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
@Override
- public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {}
+ public List<String> getServices(String clusterName) {
+ return null;
+ }
@Override
- public LogLevelFilterMap getLogLevelFilters(String clusterName) {
+ public String getGlobalConfigs(String clusterName) {
return null;
}
@Override
- public void saveOutputSolrProperties(String type, OutputSolrProperties outputSolrProperties) throws Exception {}
+ public InputConfig getInputConfig(String clusterName, String serviceName) {
+ return null;
+ }
@Override
public void close() {}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass2.java b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass2.java
index a767ff5..e94f9ce 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass2.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass2.java
@@ -22,9 +22,6 @@ package org.apache.ambari.logsearch.config.api;
import java.util.List;
import java.util.Map;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
public class LogSearchConfigServerClass2 implements LogSearchConfigServer {
@@ -40,6 +37,15 @@ public class LogSearchConfigServerClass2 implements LogSearchConfigServer {
public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
@Override
+ public LogLevelFilterManager getLogLevelFilterManager() {
+ return null;
+ }
+
+ @Override
+ public void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager) {
+ }
+
+ @Override
public void setInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
@Override
@@ -58,19 +64,5 @@ public class LogSearchConfigServerClass2 implements LogSearchConfigServer {
}
@Override
- public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) {}
-
- @Override
- public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {}
-
- @Override
- public LogLevelFilterMap getLogLevelFilters(String clusterName) {
- return null;
- }
-
- @Override
- public void saveOutputSolrProperties(String type, OutputSolrProperties outputSolrProperties) throws Exception {}
-
- @Override
public void close() {}
}
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml b/ambari-logsearch/ambari-logsearch-config-json/pom.xml
similarity index 71%
copy from ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
copy to ambari-logsearch/ambari-logsearch-config-json/pom.xml
index 145b17e..cdccbba 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-config-json/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -15,9 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
<parent>
<artifactId>ambari-logsearch</artifactId>
<groupId>org.apache.ambari</groupId>
@@ -25,9 +25,9 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>ambari-logsearch-config-zookeeper</artifactId>
+ <artifactId>ambari-logsearch-config-json</artifactId>
<packaging>jar</packaging>
- <name>Ambari Logsearch Config Zookeeper</name>
+ <name>Ambari Logsearch Config JSON</name>
<url>http://maven.apache.org</url>
<properties>
@@ -51,25 +51,6 @@
<version>3.2.2</version>
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>2.12.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <version>2.12.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>2.12.0</version>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.20</version>
@@ -80,4 +61,5 @@
<version>2.6.2</version>
</dependency>
</dependencies>
-</project>
+
+</project>
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/JsonHelper.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/JsonHelper.java
new file mode 100644
index 0000000..75fc71e
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/JsonHelper.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.json;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+import java.util.Map;
+
+/**
+ * Utility class to help JSON operations.
+ */
+public class JsonHelper {
+
+ private JsonHelper() {
+ }
+
+ public static JsonElement mergeGlobalConfigWithInputConfig(JsonParser parser, String inputConfig, JsonArray globalConfigNode) {
+ JsonElement inputConfigJson = parser.parse(inputConfig);
+ for (Map.Entry<String, JsonElement> typeEntry : inputConfigJson.getAsJsonObject().entrySet()) {
+ for (JsonElement e : typeEntry.getValue().getAsJsonArray()) {
+ for (JsonElement globalConfig : globalConfigNode) {
+ merge(globalConfig.getAsJsonObject(), e.getAsJsonObject());
+ }
+ }
+ }
+ return inputConfigJson;
+ }
+
+ public static void merge(JsonObject source, JsonObject target) {
+ for (Map.Entry<String, JsonElement> e : source.entrySet()) {
+ if (!target.has(e.getKey())) {
+ target.add(e.getKey(), e.getValue());
+ } else {
+ if (e.getValue().isJsonObject()) {
+ JsonObject valueJson = (JsonObject)e.getValue();
+ merge(valueJson, target.get(e.getKey()).getAsJsonObject());
+ }
+ }
+ }
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/ConditionsImpl.java
similarity index 95%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/ConditionsImpl.java
index 2ba472c..a6de00c 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/ConditionsImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FieldsImpl.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FieldsImpl.java
index 32a0348..426984a 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FieldsImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.util.Set;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterAdapter.java
similarity index 95%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterAdapter.java
index b84403b..fcc7876 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterAdapter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.lang.reflect.Type;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterDescriptorImpl.java
similarity index 98%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterDescriptorImpl.java
index eb9d38c..b40c139 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.util.List;
import java.util.Map;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterGrokDescriptorImpl.java
similarity index 98%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterGrokDescriptorImpl.java
index 9823163..520af97 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterGrokDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterJsonDescriptorImpl.java
similarity index 92%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterJsonDescriptorImpl.java
index 9bf1a2b..2dc8439 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterJsonDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterJsonDescriptor;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
similarity index 97%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
index 1c782c5..b45f2a0 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputAdapter.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputAdapter.java
index 5fcaf31..aa581e1 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputAdapter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.lang.reflect.Type;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputConfigGson.java
similarity index 93%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputConfigGson.java
index a7d1333..78d3db3 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputConfigGson.java
@@ -17,14 +17,14 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.lang.reflect.Type;
import java.util.List;
-import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
/**
* Helper class to convert between json string and InputConfig class.
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputConfigImpl.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputConfigImpl.java
index 6ce634f..56660af 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputConfigImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.util.List;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputCustomDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputCustomDescriptorImpl.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputCustomDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputCustomDescriptorImpl.java
index 41128c5..cc9a36f 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputCustomDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputCustomDescriptorImpl.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java
similarity index 99%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java
index 40886be..3a04ebc 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.util.Map;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
similarity index 97%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
index 8281daa..dca47f8 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputFileDescriptorImpl.java
similarity index 98%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputFileDescriptorImpl.java
index 2ba53e6..a0613d8 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputFileDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputS3FileDescriptorImpl.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputS3FileDescriptorImpl.java
index 19f52d3..527dae8 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputS3FileDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
similarity index 97%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
index 8c128de..4b67d5f 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapCustomDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapCustomDescriptorImpl.java
similarity index 97%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapCustomDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapCustomDescriptorImpl.java
index 0cb04fb..34abdb9 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapCustomDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapCustomDescriptorImpl.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapDateDescriptorImpl.java
similarity index 97%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapDateDescriptorImpl.java
index feec4b6..13869ca 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapDateDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
index ca7a6aa..fa4c436 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldDescriptorImpl.java
similarity index 95%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldDescriptorImpl.java
index 101e0d4..4581d62 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldDescriptor;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
similarity index 96%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
index 7fdaa21..5c994c9 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
similarity index 97%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
index ba4686f..8e850f2 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/PostMapValuesAdapter.java
similarity index 98%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/PostMapValuesAdapter.java
index f1ed2bd..50ce358 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/PostMapValuesAdapter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.lang.reflect.Type;
import java.util.ArrayList;
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/PostMapValuesImpl.java
similarity index 94%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesImpl.java
rename to ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/PostMapValuesImpl.java
index 4d2254a..6d6e29f 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/PostMapValuesImpl.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
import java.util.List;
diff --git a/ambari-logsearch/ambari-logsearch-config-local/pom.xml b/ambari-logsearch/ambari-logsearch-config-local/pom.xml
new file mode 100644
index 0000000..7e50f7c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-local/pom.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>ambari-logsearch</artifactId>
+ <groupId>org.apache.ambari</groupId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>ambari-logsearch-config-local</artifactId>
+ <packaging>jar</packaging>
+ <name>Ambari Logsearch Config Local</name>
+ <url>http://maven.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-config-json</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogLevelFilterManagerLocal.java b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogLevelFilterManagerLocal.java
new file mode 100644
index 0000000..5ca210d
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogLevelFilterManagerLocal.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.local;
+
+import com.google.gson.Gson;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Local implementation of Log Level Filter manager - keep the data in [config-dir]/filters folder in [service_name]-filter.json files
+ */
+public class LogLevelFilterManagerLocal implements LogLevelFilterManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterManagerLocal.class);
+
+ private final String configDir;
+ private final Gson gson;
+
+ private final FilenameFilter filterConfigFilenameFilter = (dir, name) -> name.endsWith("-filter.json");
+
+ public LogLevelFilterManagerLocal(String configDir, Gson gson) {
+ this.configDir = configDir;
+ this.gson = gson;
+ }
+
+ @Override
+ public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) throws Exception {
+ Path filterDirs = Paths.get(configDir, "filters");
+ if (!filterDirs.toFile().exists()) {
+ Files.createDirectory(filterDirs);
+ }
+ String logLevelFilterJson = gson.toJson(filter);
+ Path filePath = Paths.get(filterDirs.toAbsolutePath().toString(), String.format("%s-filter.json", logId.toLowerCase()));
+ byte[] data = logLevelFilterJson.getBytes(StandardCharsets.UTF_8);
+ Files.write(filePath, data);
+ }
+
+ @Override
+ public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {
+ for (Map.Entry<String, LogLevelFilter> e : filters.getFilter().entrySet()) {
+ Path filterDirs = Paths.get(configDir, "filters");
+ String logLevelFilterJson = gson.toJson(e.getValue());
+ Path filePath = Paths.get(filterDirs.toAbsolutePath().toString(), String.format("%s-filter.json", e.getKey()));
+ if (filePath.toFile().exists()) {
+ String currentLogLevelFilterJson = new String(Files.readAllBytes(filePath));
+ if (!logLevelFilterJson.equals(currentLogLevelFilterJson)) {
+ byte[] data = logLevelFilterJson.getBytes(StandardCharsets.UTF_8);
+ Files.write(filePath, data);
+ LOG.info("Set log level filter for the log " + e.getKey() + " for cluster " + clusterName);
+ }
+ }
+ }
+ }
+
+ @Override
+ public LogLevelFilterMap getLogLevelFilters(String clusterName) {
+ TreeMap<String, LogLevelFilter> filters = new TreeMap<>();
+ File filterDirs = Paths.get(configDir, "filters").toFile();
+ if (filterDirs.exists()) {
+ File[] logLevelFilterFiles = filterDirs.listFiles(filterConfigFilenameFilter);
+ if (logLevelFilterFiles != null) {
+ for (File file : logLevelFilterFiles) {
+ try {
+ String serviceName = file.getName().replace("-filter.json", "").toLowerCase();
+ String logLevelFilterStr = new String(Files.readAllBytes(file.toPath()));
+ LogLevelFilter logLevelFilter = gson.fromJson(logLevelFilterStr, LogLevelFilter.class);
+ filters.put(serviceName, logLevelFilter);
+ } catch (IOException e) {
+ // skip
+ }
+ }
+ }
+ }
+ LogLevelFilterMap logLevelFilters = new LogLevelFilterMap();
+ logLevelFilters.setFilter(filters);
+ return logLevelFilters;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLocal.java b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLocal.java
new file mode 100644
index 0000000..191b6f8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLocal.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.local;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
+import org.apache.ambari.logsearch.config.api.LogSearchConfig;
+
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.regex.Pattern;
+
+/**
+ * Abstract local config location handler - holds common operations for Log Search Server and Log Feeder local config handler
+ */
+public abstract class LogSearchConfigLocal implements LogSearchConfig {
+
+ private static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+
+ protected Map<String, String> properties;
+ protected final Map<String, Map<String, String>> clusterInputConfigMap = new ConcurrentHashMap<>();
+ protected final Map<String, String> inputFileContentsMap = new ConcurrentHashMap<>();
+ protected Gson gson;
+ protected final FilenameFilter inputConfigFileFilter = (dir, name) -> name.startsWith("input.config-") && name.endsWith(".json");
+ protected final Pattern serviceNamePattern = Pattern.compile("input.config-(.+).json");
+ protected final ExecutorService executorService = Executors.newCachedThreadPool();
+ protected LogLevelFilterManager logLevelFilterManager;
+
+ public void init(Map<String, String> properties) throws Exception {
+ this.properties = properties;
+ gson = new GsonBuilder().setDateFormat(DATE_FORMAT).setPrettyPrinting().create();
+ }
+
+ @Override
+ public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {
+ Path filePath = Paths.get(getConfigDir(), String.format("input.config-%s.json", serviceName.toLowerCase()));
+ byte[] data = inputConfig.getBytes(StandardCharsets.UTF_8);
+ Files.write(filePath, data);
+ inputFileContentsMap.put(filePath.toAbsolutePath().toString(), inputConfig);
+ if (!clusterInputConfigMap.containsKey(clusterName)) {
+ clusterInputConfigMap.put(clusterName, inputFileContentsMap);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ public abstract String getConfigDir();
+
+ public abstract void setConfigDir(String configDir);
+
+ @Override
+ public LogLevelFilterManager getLogLevelFilterManager() {
+ return logLevelFilterManager;
+ }
+
+ @Override
+ public void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager) {
+ this.logLevelFilterManager = logLevelFilterManager;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLocalUpdater.java b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLocalUpdater.java
new file mode 100644
index 0000000..76379ec
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLocalUpdater.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.local;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
+import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.ambari.logsearch.config.json.JsonHelper;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.ClosedWatchServiceException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
+
+/**
+ * Watch specific config folder, and check against input.config-*.json file changes (create/update/remove),
+ * a change can trigger an input config monitor (which should start to monitor input files with the new or updated settings)
+ */
+public class LogSearchConfigLocalUpdater implements Runnable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogSearchConfigLocalUpdater.class);
+
+ private final Path path;
+ private final WatchService watchService;
+ private final InputConfigMonitor inputConfigMonitor;
+ private final Map<String, String> inputFileContentsMap;
+ private final JsonParser parser;
+ private final JsonArray globalConfigNode;
+ private final Pattern serviceNamePattern;
+
+ public LogSearchConfigLocalUpdater(final Path path, final WatchService watchService,
+ final InputConfigMonitor inputConfigMonitor, final Map<String, String> inputFileContentsMap,
+ final JsonParser parser, final JsonArray globalConfigNode, final Pattern serviceNamePattern) {
+ this.path = path;
+ this.watchService = watchService;
+ this.inputConfigMonitor = inputConfigMonitor;
+ this.inputFileContentsMap = inputFileContentsMap;
+ this.parser = parser;
+ this.globalConfigNode = globalConfigNode;
+ this.serviceNamePattern = serviceNamePattern;
+ }
+
+ @Override
+ public void run() {
+ final Map<WatchKey, Path> keys = new ConcurrentHashMap<>();
+ try {
+ register(this.path, keys, watchService);
+ } catch (IOException e) {
+ LOG.error("{}", e);
+ throw new RuntimeException(e);
+ }
+ while (!Thread.interrupted()) {
+ WatchKey key;
+ try {
+ key = watchService.poll(10, TimeUnit.SECONDS);
+ } catch (InterruptedException | ClosedWatchServiceException e) {
+ break;
+ }
+ if (key != null) {
+ Path path = keys.get(key);
+ for (WatchEvent<?> ev : key.pollEvents()) {
+ WatchEvent<Path> event = cast(ev);
+ WatchEvent.Kind<Path> kind = event.kind();
+ Path name = event.context();
+ Path monitoredInput = path.resolve(name);
+ File file = monitoredInput.toFile();
+ String absPath = monitoredInput.toAbsolutePath().toString();
+ if (file.getName().startsWith("input.config-") && file.getName().endsWith(".json")) {
+ Matcher m = serviceNamePattern.matcher(file.getName());
+ m.find();
+ String serviceName = m.group(1);
+ try {
+ if (kind == ENTRY_CREATE) {
+ LOG.info("New input config entry found: {}", absPath);
+ String inputConfig = new String(Files.readAllBytes(monitoredInput));
+ JsonElement inputConfigJson = JsonHelper.mergeGlobalConfigWithInputConfig(parser, inputConfig, globalConfigNode);
+ inputConfigMonitor.loadInputConfigs(serviceName, InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+ inputFileContentsMap.put(absPath, inputConfig);
+ } else if (kind == ENTRY_MODIFY) {
+ LOG.info("Input config entry modified: {}", absPath);
+ if (inputFileContentsMap.containsKey(absPath)) {
+ String oldContent = inputFileContentsMap.get(absPath);
+ String inputConfig = new String(Files.readAllBytes(monitoredInput));
+ if (!inputConfig.equals(oldContent)) {
+ inputConfigMonitor.removeInputs(serviceName);
+ inputFileContentsMap.remove(absPath);
+ JsonElement inputConfigJson = JsonHelper.mergeGlobalConfigWithInputConfig(parser, inputConfig, globalConfigNode);
+ inputConfigMonitor.loadInputConfigs(serviceName, InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+ inputFileContentsMap.put(absPath, inputConfig);
+ }
+ }
+ } else if (kind == ENTRY_DELETE) {
+ LOG.info("Input config deleted: {}", absPath);
+ if (inputFileContentsMap.containsKey(absPath)) {
+ inputConfigMonitor.removeInputs(serviceName);
+ inputFileContentsMap.remove(absPath);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("{}", e);
+ break;
+ }
+ }
+ }
+ if (!key.reset()) {
+ LOG.info("{} is invalid", key);
+ keys.remove(key);
+ if (keys.isEmpty()) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private void register(Path dir, Map<WatchKey, Path> keys, WatchService watchService)
+ throws IOException {
+ WatchKey key = dir.register(watchService, ENTRY_CREATE, ENTRY_DELETE,
+ ENTRY_MODIFY);
+ keys.put(key, dir);
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> WatchEvent<T> cast(WatchEvent<?> event) {
+ return (WatchEvent<T>) event;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
new file mode 100644
index 0000000..2a02cb5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.local;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
+import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterMonitor;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.ambari.logsearch.config.json.JsonHelper;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.WatchService;
+import java.util.Map;
+import java.util.regex.Matcher;
+
+/**
+ * Handle input.config-*.json files from local sourse (filesystem)
+ * After the first file check in the configuration folder, it starts to watch the specified about changes (create/update/delete files)
+ */
+public class LogSearchConfigLogFeederLocal extends LogSearchConfigLocal implements LogSearchConfigLogFeeder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogSearchConfigLogFeederLocal.class);
+
+ private String configDir;
+
+ @Override
+ public void init(Map<String, String> properties, String clusterName) throws Exception {
+ super.init(properties);
+ setConfigDir(properties.getOrDefault("logfeeder.config.dir", "/usr/lib/ambari-logsearch-logfeeder/conf"));
+ boolean localConfig = Boolean.valueOf(properties.getOrDefault("logfeeder.config.filter.local", "false"));
+ if (localConfig) {
+ setLogLevelFilterManager(new LogLevelFilterManagerLocal(getConfigDir(), gson));
+ }
+ }
+
+ @Override
+ public boolean inputConfigExists(String serviceName) throws Exception {
+ Path filePath = Paths.get(getConfigDir(), String.format("input.config-%s.json", serviceName.toLowerCase()));
+ return inputFileContentsMap.containsKey(filePath.toAbsolutePath().toString());
+ }
+
+ @Override
+ public void monitorInputConfigChanges(final InputConfigMonitor inputConfigMonitor, final LogLevelFilterMonitor logLevelFilterMonitor, String clusterName) throws Exception {
+ final JsonParser parser = new JsonParser();
+ final JsonArray globalConfigNode = new JsonArray();
+ for (String globalConfigJsonString : inputConfigMonitor.getGlobalConfigJsons()) {
+ JsonElement globalConfigJson = parser.parse(globalConfigJsonString);
+ globalConfigNode.add(globalConfigJson.getAsJsonObject().get("global"));
+ Path filePath = Paths.get(configDir, "global.config.json");
+ String strData = InputConfigGson.gson.toJson(globalConfigJson);
+ byte[] data = strData.getBytes(StandardCharsets.UTF_8);
+ Files.write(filePath, data);
+ }
+
+ File[] inputConfigFiles = new File(configDir).listFiles(inputConfigFileFilter);
+ if (inputConfigFiles != null) {
+ for (File inputConfigFile : inputConfigFiles) {
+ String inputConfig = new String(Files.readAllBytes(inputConfigFile.toPath()));
+ Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
+ m.find();
+ String serviceName = m.group(1);
+ JsonElement inputConfigJson = JsonHelper.mergeGlobalConfigWithInputConfig(parser, inputConfig, globalConfigNode);
+ inputConfigMonitor.loadInputConfigs(serviceName, InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+ }
+ }
+ final FileSystem fs = FileSystems.getDefault();
+ final WatchService ws = fs.newWatchService();
+ Path configPath = Paths.get(configDir);
+ LogSearchConfigLocalUpdater updater = new LogSearchConfigLocalUpdater(configPath, ws, inputConfigMonitor, inputFileContentsMap,
+ parser, globalConfigNode, serviceNamePattern);
+ executorService.submit(updater);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public String getConfigDir() {
+ return this.configDir;
+ }
+
+ @Override
+ public void setConfigDir(String configDir) {
+ this.configDir = configDir;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigServerLocal.java
similarity index 56%
copy from ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java
copy to ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigServerLocal.java
index 71e924a..13aacd0 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigServerClass1.java
+++ b/ambari-logsearch/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigServerLocal.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,35 +16,48 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.ambari.logsearch.config.local;
-package org.apache.ambari.logsearch.config.api;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
+import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
+import java.io.File;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
-import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
+// TODO: implement every method of this, although that can be useful only for local 1-node deployments
+public class LogSearchConfigServerLocal extends LogSearchConfigLocal implements LogSearchConfigServer {
-public class LogSearchConfigServerClass1 implements LogSearchConfigServer {
- @Override
- public void init(Map<String, String> properties) {}
+ private String configDir;
@Override
- public boolean inputConfigExists(String clusterName, String serviceName) throws Exception {
- return false;
+ public void init(Map<String, String> properties) throws Exception {
+ super.init(properties);
+ setConfigDir(properties.getOrDefault("logsearch.logfeeder.config.dir", "/usr/lib/ambari-logsearch-portal/conf/input-configs"));
+ File confDirFile = new File(configDir);
+ if (!confDirFile.exists()) {
+ confDirFile.mkdir();
+ }
+ boolean localConfig = Boolean.valueOf(properties.getOrDefault("logsearch.logfeeder.config.filter.local", "false"));
+ if (localConfig) {
+ setLogLevelFilterManager(new LogLevelFilterManagerLocal(getConfigDir(), gson));
+ }
}
@Override
- public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
+ public List<String> getServices(String clusterName) {
+ return null;
+ }
@Override
- public void setInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {}
+ public boolean inputConfigExists(String clusterName, String serviceName) throws Exception {
+ return false;
+ }
@Override
- public List<String> getServices(String clusterName) {
- return null;
+ public void setInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {
}
@Override
@@ -58,19 +71,20 @@ public class LogSearchConfigServerClass1 implements LogSearchConfigServer {
}
@Override
- public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) {}
-
- @Override
- public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {}
+ public void createInputConfig(String clusterName, String serviceName, String inputConfig) throws Exception {
+ }
@Override
- public LogLevelFilterMap getLogLevelFilters(String clusterName) {
- return null;
+ public void close() throws IOException {
}
@Override
- public void saveOutputSolrProperties(String type, OutputSolrProperties outputSolrProperties) throws Exception {}
+ public String getConfigDir() {
+ return this.configDir;
+ }
@Override
- public void close() {}
-}
\ No newline at end of file
+ public void setConfigDir(String configDir) {
+ this.configDir = configDir;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml b/ambari-logsearch/ambari-logsearch-config-solr/pom.xml
similarity index 52%
copy from ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
copy to ambari-logsearch/ambari-logsearch-config-solr/pom.xml
index 145b17e..f47fd75 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-config-solr/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -15,9 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
<parent>
<artifactId>ambari-logsearch</artifactId>
<groupId>org.apache.ambari</groupId>
@@ -25,9 +25,9 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>ambari-logsearch-config-zookeeper</artifactId>
+ <artifactId>ambari-logsearch-config-solr</artifactId>
<packaging>jar</packaging>
- <name>Ambari Logsearch Config Zookeeper</name>
+ <name>Ambari Logsearch Config Solr</name>
<url>http://maven.apache.org</url>
<properties>
@@ -41,43 +41,15 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.4</version>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <version>3.2.2</version>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>2.12.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <version>2.12.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>2.12.0</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.20</version>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-config-json</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.6.2</version>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
</dependency>
</dependencies>
-</project>
+
+</project>
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-config-solr/src/main/java/org/apache/ambari/logsearch/config/solr/LogLevelFilterManagerSolr.java b/ambari-logsearch/ambari-logsearch-config-solr/src/main/java/org/apache/ambari/logsearch/config/solr/LogLevelFilterManagerSolr.java
new file mode 100644
index 0000000..8d8976b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-solr/src/main/java/org/apache/ambari/logsearch/config/solr/LogLevelFilterManagerSolr.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.solr;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Gather and store log level filters from/in a Solr collection.
+ */
+public class LogLevelFilterManagerSolr implements LogLevelFilterManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterManagerSolr.class);
+
+ private static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+ private final SolrClient solrClient;
+ private Gson gson;
+ private boolean useClusterParam = true;
+
+ public LogLevelFilterManagerSolr(SolrClient solrClient) {
+ this.solrClient = solrClient;
+ waitForSolr(solrClient);
+ gson = new GsonBuilder().setDateFormat(DATE_FORMAT).setPrettyPrinting().create();
+ }
+
+ @Override
+ public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) throws Exception {
+ final SolrInputDocument doc = new SolrInputDocument();
+ int hashCode = useClusterParam ? (clusterName + logId).hashCode() : logId.hashCode();
+ doc.addField("id", String.valueOf(hashCode));
+ if (useClusterParam) {
+ doc.addField("cluster_string", clusterName);
+ }
+ doc.addField("filtername", logId);
+ doc.addField("rowtype", "log_level_filter");
+ doc.addField("jsons", gson.toJson(filter));
+ doc.addField("username", "default");
+ LOG.debug("Creating log level filter - logid: {}, cluster: {}", logId, clusterName);
+ solrClient.add(doc);
+ }
+
+ @Override
+ public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {
+ TreeMap<String, LogLevelFilter> logLevelFilterTreeMap = filters.getFilter();
+ if (!logLevelFilterTreeMap.isEmpty()) {
+ LogLevelFilterMap actualFiltersMap = getLogLevelFilters(clusterName);
+ if (actualFiltersMap.getFilter().isEmpty()) {
+ if (!filters.getFilter().isEmpty()) {
+ for (Map.Entry<String, LogLevelFilter> entry : filters.getFilter().entrySet()) {
+ createLogLevelFilter(clusterName, entry.getKey(), entry.getValue());
+ }
+ }
+ } else {
+ TreeMap<String, LogLevelFilter> mapToSet = filters.getFilter();
+ TreeMap<String, LogLevelFilter> finalMapToSet = new TreeMap<>();
+ for (Map.Entry<String, LogLevelFilter> entry : actualFiltersMap.getFilter().entrySet()) {
+ if (mapToSet.containsKey(entry.getKey())) {
+ String actualValue = gson.toJson(entry.getValue());
+ String newValue = gson.toJson(mapToSet.get(entry.getKey()));
+ if (!newValue.equals(actualValue)) {
+ finalMapToSet.put(entry.getKey(), mapToSet.get(entry.getKey()));
+ }
+ } else {
+ finalMapToSet.put(entry.getKey(), mapToSet.get(entry.getKey()));
+ }
+ }
+ for (Map.Entry<String, LogLevelFilter> entry : finalMapToSet.entrySet()) {
+ createLogLevelFilter(clusterName, entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ }
+
+ @Override
+ public LogLevelFilterMap getLogLevelFilters(String clusterName) {
+ LogLevelFilterMap logLevelFilterMap = new LogLevelFilterMap();
+ TreeMap<String, LogLevelFilter> logLevelFilterTreeMap = new TreeMap<>();
+ try {
+ SolrQuery solrQuery = new SolrQuery();
+ solrQuery.setQuery("*:*");
+ if (useClusterParam) {
+ solrQuery.addFilterQuery("cluster_string:" + clusterName);
+ }
+ solrQuery.addFilterQuery("rowtype:log_level_filter");
+ solrQuery.setFields("jsons", "filtername");
+
+ final QueryResponse response = solrClient.query(solrQuery);
+ if (response != null) {
+ final SolrDocumentList documents = response.getResults();
+ if (documents != null && !documents.isEmpty()) {
+ for(SolrDocument document : documents) {
+ String jsons = (String) document.getFieldValue("jsons");
+ String logId = (String) document.getFieldValue("filtername");
+ if (jsons != null) {
+ LogLevelFilter logLevelFilter = gson.fromJson(jsons, LogLevelFilter.class);
+ logLevelFilterTreeMap.put(logId,logLevelFilter);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error during getting log level filters: {}", e);
+ }
+ logLevelFilterMap.setFilter(logLevelFilterTreeMap);
+ return logLevelFilterMap;
+ }
+
+ public boolean isUseClusterParam() {
+ return useClusterParam;
+ }
+
+ public void setUseClusterParam(boolean useClusterParam) {
+ this.useClusterParam = useClusterParam;
+ }
+
+ public Gson getGson() {
+ return gson;
+ }
+
+ private void waitForSolr(SolrClient solrClient) {
+ boolean solrAvailable = false;
+ while (!solrAvailable) {
+ try {
+ LOG.debug("Start solr ping for log level filter collection");
+ SolrPingResponse pingResponse = solrClient.ping();
+ if (pingResponse.getStatus() == 0) {
+ solrAvailable = true;
+ break;
+ }
+ } catch (Exception e) {
+ LOG.error("{}", e);
+ }
+ LOG.info("Solr (collection for log level filters) is not available yet. Sleeping 10 sec. Retrying...");
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ LOG.error("{}", e);
+ }
+ }
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-solr/src/main/java/org/apache/ambari/logsearch/config/solr/LogLevelFilterUpdaterSolr.java b/ambari-logsearch/ambari-logsearch-config-solr/src/main/java/org/apache/ambari/logsearch/config/solr/LogLevelFilterUpdaterSolr.java
new file mode 100644
index 0000000..534a60c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-solr/src/main/java/org/apache/ambari/logsearch/config/solr/LogLevelFilterUpdaterSolr.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.solr;
+
+import com.google.gson.Gson;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterMonitor;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterUpdater;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Periodically checks log level filters in Solr, and send a notification about any change to a log level filter monitor.
+ */
+public class LogLevelFilterUpdaterSolr extends LogLevelFilterUpdater {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterUpdaterSolr.class);
+
+ private final LogLevelFilterManagerSolr logLevelFilterManagerSolr;
+ private final String cluster;
+
+ public LogLevelFilterUpdaterSolr(String threadName, LogLevelFilterMonitor logLevelFilterMonitor, Integer interval,
+ LogLevelFilterManagerSolr logLevelFilterManagerSolr, String cluster) {
+ super(threadName, logLevelFilterMonitor, interval);
+ this.logLevelFilterManagerSolr = logLevelFilterManagerSolr;
+ this.cluster = cluster;
+ }
+
+ @Override
+ protected void checkFilters(LogLevelFilterMonitor logLevelFilterMonitor) {
+ try {
+ LOG.debug("Start checking log level filters in Solr ...");
+ LogLevelFilterMap logLevelFilterMap = logLevelFilterManagerSolr.getLogLevelFilters(cluster);
+ Map<String, LogLevelFilter> filters = logLevelFilterMap.getFilter();
+ Map<String, LogLevelFilter> copiedStoredFilters = new ConcurrentHashMap<>(logLevelFilterMonitor.getLogLevelFilters());
+ final Gson gson = logLevelFilterManagerSolr.getGson();
+ for (Map.Entry<String, LogLevelFilter> logFilterEntry : filters.entrySet()){
+ if (copiedStoredFilters.containsKey(logFilterEntry.getKey())) {
+ String remoteValue = gson.toJson(logFilterEntry.getValue());
+ String storedValue = gson.toJson(copiedStoredFilters.get(logFilterEntry.getKey()));
+ if (!storedValue.equals(remoteValue)) {
+ LOG.info("Log level filter updated for {}", logFilterEntry.getKey());
+ logLevelFilterMonitor.setLogLevelFilter(logFilterEntry.getKey(), logFilterEntry.getValue());
+ }
+ } else {
+ LOG.info("New log level filter registered: {}", logFilterEntry.getKey());
+ logLevelFilterMonitor.setLogLevelFilter(logFilterEntry.getKey(), logFilterEntry.getValue());
+ }
+ }
+ for (Map.Entry<String, LogLevelFilter> storedLogFilterEntry : copiedStoredFilters.entrySet()) {
+ if (!filters.containsKey(storedLogFilterEntry.getKey())) {
+ LOG.info("Removing log level filter: {}", storedLogFilterEntry.getKey());
+ logLevelFilterMonitor.removeLogLevelFilter(storedLogFilterEntry.getKey());
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error during filter Solr check: {}",e);
+ }
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml b/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
index 145b17e..343971e 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml
@@ -37,20 +37,10 @@
<dependencies>
<dependency>
<groupId>org.apache.ambari</groupId>
- <artifactId>ambari-logsearch-config-api</artifactId>
+ <artifactId>ambari-logsearch-config-json</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.4</version>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <version>3.2.2</version>
- </dependency>
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
@@ -69,15 +59,5 @@
<artifactId>curator-recipes</artifactId>
<version>2.12.0</version>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.20</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.6.2</version>
- </dependency>
</dependencies>
</project>
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java
new file mode 100644
index 0000000..1ad7517
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.zookeeper;
+
+import com.google.gson.Gson;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
+import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class LogLevelFilterManagerZK implements LogLevelFilterManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterManagerZK.class);
+
+ private final CuratorFramework client;
+ private final TreeCache serverCache;
+ private final Gson gson;
+ private final List<ACL> aclList;
+
+ public LogLevelFilterManagerZK(CuratorFramework client, TreeCache serverCache, List<ACL> aclList, Gson gson) {
+ this.client = client;
+ this.serverCache = serverCache;
+ this.aclList = aclList;
+ this.gson = gson;
+ }
+
+ @Override
+ public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) throws Exception {
+ String nodePath = String.format("/%s/loglevelfilter/%s", clusterName, logId);
+ String logLevelFilterJson = gson.toJson(filter);
+ try {
+ client.create().creatingParentContainersIfNeeded().withACL(aclList).forPath(nodePath, logLevelFilterJson.getBytes());
+ LOG.info("Uploaded log level filter for the log " + logId + " for cluster " + clusterName);
+ } catch (KeeperException.NodeExistsException e) {
+ LOG.debug("Did not upload log level filters for log " + logId + " as it was already uploaded by another Log Feeder");
+ }
+ }
+
+ @Override
+ public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {
+ for (Map.Entry<String, LogLevelFilter> e : filters.getFilter().entrySet()) {
+ String nodePath = String.format("/%s/loglevelfilter/%s", clusterName, e.getKey());
+ String logLevelFilterJson = gson.toJson(e.getValue());
+ String currentLogLevelFilterJson = new String(serverCache.getCurrentData(nodePath).getData());
+ if (!logLevelFilterJson.equals(currentLogLevelFilterJson)) {
+ client.setData().forPath(nodePath, logLevelFilterJson.getBytes());
+ LOG.info("Set log level filter for the log " + e.getKey() + " for cluster " + clusterName);
+ }
+ }
+ }
+
+ @Override
+ public LogLevelFilterMap getLogLevelFilters(String clusterName) {
+ String parentPath = String.format("/%s/loglevelfilter", clusterName);
+ TreeMap<String, LogLevelFilter> filters = new TreeMap<>();
+ Map<String, ChildData> logLevelFilterNodes = serverCache.getCurrentChildren(parentPath);
+ if (logLevelFilterNodes != null && !logLevelFilterNodes.isEmpty()) {
+ for (Map.Entry<String, ChildData> e : logLevelFilterNodes.entrySet()) {
+ LogLevelFilter logLevelFilter = gson.fromJson(new String(e.getValue().getData()), LogLevelFilter.class);
+ filters.put(e.getKey(), logLevelFilter);
+ }
+ }
+ LogLevelFilterMap logLevelFilters = new LogLevelFilterMap();
+ logLevelFilters.setFilter(filters);
+ return logLevelFilters;
+ }
+
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigLogFeederZK.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigLogFeederZK.java
index c050540..8082ba7 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigLogFeederZK.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigLogFeederZK.java
@@ -19,21 +19,18 @@
package org.apache.ambari.logsearch.config.zookeeper;
-import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
-import org.apache.ambari.logsearch.config.api.OutputConfigMonitor;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
-import org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl.OutputSolrPropertiesImpl;
+import org.apache.ambari.logsearch.config.json.JsonHelper;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
import org.apache.ambari.logsearch.config.api.LogLevelFilterMonitor;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent.Type;
@@ -45,7 +42,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableSet;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
public class LogSearchConfigLogFeederZK extends LogSearchConfigZK implements LogSearchConfigLogFeeder {
@@ -64,6 +60,8 @@ public class LogSearchConfigLogFeederZK extends LogSearchConfigZK implements Log
}
logFeederClusterCache = new TreeCache(client, String.format("/%s", clusterName));
+ LogLevelFilterManager logLevelFilterManager = new LogLevelFilterManagerZK(client, null, getAcls(), gson);
+ setLogLevelFilterManager(logLevelFilterManager);
}
@Override
@@ -135,7 +133,7 @@ public class LogSearchConfigLogFeederZK extends LogSearchConfigZK implements Log
for (Map.Entry<String, JsonElement> typeEntry : inputConfigJson.getAsJsonObject().entrySet()) {
for (JsonElement e : typeEntry.getValue().getAsJsonArray()) {
for (JsonElement globalConfig : globalConfigNode) {
- merge(globalConfig.getAsJsonObject(), e.getAsJsonObject());
+ JsonHelper.merge(globalConfig.getAsJsonObject(), e.getAsJsonObject());
}
}
}
@@ -162,19 +160,6 @@ public class LogSearchConfigLogFeederZK extends LogSearchConfigZK implements Log
break;
}
}
-
- private void merge(JsonObject source, JsonObject target) {
- for (Map.Entry<String, JsonElement> e : source.entrySet()) {
- if (!target.has(e.getKey())) {
- target.add(e.getKey(), e.getValue());
- } else {
- if (e.getValue().isJsonObject()) {
- JsonObject valueJson = (JsonObject)e.getValue();
- merge(valueJson, target.get(e.getKey()).getAsJsonObject());
- }
- }
- }
- }
};
logFeederClusterCache.getListenable().addListener(listener);
logFeederClusterCache.start();
@@ -194,35 +179,4 @@ public class LogSearchConfigLogFeederZK extends LogSearchConfigZK implements Log
LOG.warn("Exception during global config node creation/update", e);
}
}
-
- @Override
- public OutputSolrProperties getOutputSolrProperties(String type) throws Exception {
- String nodePath = String.format("/output/solr/%s", type);
- ChildData currentData = outputCache.getCurrentData(nodePath);
- return currentData == null ?
- null :
- gson.fromJson(new String(currentData.getData()), OutputSolrPropertiesImpl.class);
- }
-
- @Override
- public void monitorOutputProperties(final List<? extends OutputConfigMonitor> outputConfigMonitors) throws Exception {
- TreeCacheListener listener = new TreeCacheListener() {
- public void childEvent(CuratorFramework client, TreeCacheEvent event) throws Exception {
- if (event.getType() != Type.NODE_UPDATED) {
- return;
- }
-
- LOG.info("Output config updated: " + event.getData().getPath());
- for (OutputConfigMonitor monitor : outputConfigMonitors) {
- String monitorPath = String.format("/output/%s/%s", monitor.getDestination(), monitor.getOutputType());
- if (monitorPath.equals(event.getData().getPath())) {
- String nodeData = new String(event.getData().getData());
- OutputSolrProperties outputSolrProperties = gson.fromJson(nodeData, OutputSolrPropertiesImpl.class);
- monitor.outputConfigChanged(outputSolrProperties);
- }
- }
- }
- };
- outputCache.getListenable().addListener(listener);
- }
}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigServerZK.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigServerZK.java
index 9973196..d8fd79b 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigServerZK.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigServerZK.java
@@ -24,14 +24,14 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputAdapter;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.slf4j.Logger;
@@ -57,6 +57,8 @@ public class LogSearchConfigServerZK extends LogSearchConfigZK implements LogSea
}
serverCache = new TreeCache(client, "/");
serverCache.start();
+ LogLevelFilterManager logLevelFilterManager = new LogLevelFilterManagerZK(client, serverCache, getAcls(), gson);
+ setLogLevelFilterManager(logLevelFilterManager);
}
@Override
@@ -97,42 +99,4 @@ public class LogSearchConfigServerZK extends LogSearchConfigZK implements LogSea
return childData == null ? null : InputConfigGson.gson.fromJson(new String(childData.getData()), InputConfigImpl.class);
}
- @Override
- public void setLogLevelFilters(String clusterName, LogLevelFilterMap filters) throws Exception {
- for (Map.Entry<String, LogLevelFilter> e : filters.getFilter().entrySet()) {
- String nodePath = String.format("/%s/loglevelfilter/%s", clusterName, e.getKey());
- String logLevelFilterJson = gson.toJson(e.getValue());
- String currentLogLevelFilterJson = new String(serverCache.getCurrentData(nodePath).getData());
- if (!logLevelFilterJson.equals(currentLogLevelFilterJson)) {
- client.setData().forPath(nodePath, logLevelFilterJson.getBytes());
- LOG.info("Set log level filter for the log " + e.getKey() + " for cluster " + clusterName);
- }
- }
- }
-
- @Override
- public LogLevelFilterMap getLogLevelFilters(String clusterName) {
- String parentPath = String.format("/%s/loglevelfilter", clusterName);
- Map<String, ChildData> logLevelFilterNodes = serverCache.getCurrentChildren(parentPath);
- TreeMap<String, LogLevelFilter> filters = new TreeMap<>();
- for (Map.Entry<String, ChildData> e : logLevelFilterNodes.entrySet()) {
- LogLevelFilter logLevelFilter = gson.fromJson(new String(e.getValue().getData()), LogLevelFilter.class);
- filters.put(e.getKey(), logLevelFilter);
- }
-
- LogLevelFilterMap logLevelFilters = new LogLevelFilterMap();
- logLevelFilters.setFilter(filters);
- return logLevelFilters;
- }
-
- @Override
- public void saveOutputSolrProperties(String type, OutputSolrProperties outputSolrProperties) throws Exception {
- String nodePath = String.format("/output/solr/%s", type);
- String data = gson.toJson(outputSolrProperties);
- if (outputCache.getCurrentData(nodePath) == null) {
- client.create().creatingParentContainersIfNeeded().withACL(getAcls()).forPath(nodePath, data.getBytes());
- } else {
- client.setData().forPath(nodePath, data.getBytes());
- }
- }
}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java
index 382d8fa..7e51c35 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
import org.apache.ambari.logsearch.config.api.LogSearchConfig;
import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
@@ -107,6 +108,7 @@ public class LogSearchConfigZK implements LogSearchConfig {
protected CuratorFramework client;
protected TreeCache outputCache;
protected Gson gson;
+ protected LogLevelFilterManager logLevelFilterManager;
public void init(Map<String, String> properties) throws Exception {
this.properties = properties;
@@ -154,15 +156,13 @@ public class LogSearchConfigZK implements LogSearchConfig {
}
@Override
- public void createLogLevelFilter(String clusterName, String logId, LogLevelFilter filter) throws Exception {
- String nodePath = String.format("/%s/loglevelfilter/%s", clusterName, logId);
- String logLevelFilterJson = gson.toJson(filter);
- try {
- client.create().creatingParentContainersIfNeeded().withACL(getAcls()).forPath(nodePath, logLevelFilterJson.getBytes());
- LOG.info("Uploaded log level filter for the log " + logId + " for cluster " + clusterName);
- } catch (NodeExistsException e) {
- LOG.debug("Did not upload log level filters for log " + logId + " as it was already uploaded by another Log Feeder");
- }
+ public LogLevelFilterManager getLogLevelFilterManager() {
+ return this.logLevelFilterManager;
+ }
+
+ @Override
+ public void setLogLevelFilterManager(LogLevelFilterManager logLevelFilterManager) {
+ this.logLevelFilterManager = logLevelFilterManager;
}
protected List<ACL> getAcls() {
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java
index b173ff1..92a659b 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java
@@ -18,7 +18,7 @@
*/
package org.apache.ambari.logsearch.domain;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputAdapter;
import org.apache.solr.client.solrj.SolrClient;
import org.jbehave.web.selenium.WebDriverProvider;
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
index a4cc2c2..25e49d5 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java
@@ -36,7 +36,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ambari.logfeeder.common.LogEntryParseTester;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputAdapter;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java
index 1a3ef02..892806d 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java
@@ -24,8 +24,8 @@ import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.not;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
import org.apache.ambari.logsearch.domain.StoryDataRegistry;
import org.hamcrest.Matchers;
import org.jbehave.core.annotations.Then;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder-container-registry/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder-container-registry/pom.xml
index 66333f5..001d6fa 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder-container-registry/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder-container-registry/pom.xml
@@ -25,6 +25,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
+ <name>Ambari Logsearch Log Feeder Container Registry</name>
<artifactId>ambari-logsearch-logfeeder-container-registry</artifactId>
<dependencies>
<dependency>
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java
index a2f13b8..f369680 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java
@@ -26,7 +26,6 @@ import org.apache.ambari.logfeeder.plugin.common.MetricData;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
import org.apache.ambari.logsearch.config.api.OutputConfigMonitor;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,8 +47,6 @@ public abstract class Output<PROP_TYPE extends LogFeederProperties, INPUT_MARKER
public abstract String getOutputType();
- public abstract void outputConfigChanged(OutputProperties outputProperties);
-
public abstract void copyFile(File inputFile, InputMarker inputMarker) throws Exception;
public abstract void write(String jsonStr, INPUT_MARKER inputMarker) throws Exception;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index 2544c4c..2f828bc 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -45,11 +45,26 @@
</dependency>
<dependency>
<groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-config-json</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
<artifactId>ambari-logsearch-config-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-config-local</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-config-solr</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
<artifactId>ambari-logsearch-logfeeder-plugin-api</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
index 80b7104..8dbb39a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
@@ -33,12 +33,11 @@ import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
-import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterDescriptorImpl;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.FilterDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputDescriptorImpl;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.BooleanUtils;
@@ -97,8 +96,6 @@ public class ConfigHandler implements InputConfigMonitor {
inputManager.init();
outputManager.init();
-
- logSearchConfig.monitorOutputProperties(outputManager.getOutputsToMonitor());
}
private void loadConfigFiles() throws Exception {
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
index 9e4edcc..6fd4140 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
@@ -34,9 +34,9 @@ import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
+import org.apache.ambari.logsearch.config.json.JsonHelper;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonArray;
@@ -124,10 +124,6 @@ public class LogEntryParseTester {
}
@Override
- public void outputConfigChanged(OutputProperties outputProperties) {
- }
-
- @Override
public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
}
@@ -150,24 +146,11 @@ public class LogEntryParseTester {
for (JsonObject globalConfig : globalConfigs) {
for (Map.Entry<String, JsonElement> typeEntry : shipperConfigJson.getAsJsonObject().entrySet()) {
for (JsonElement e : typeEntry.getValue().getAsJsonArray()) {
- merge(globalConfig, e.getAsJsonObject());
+ JsonHelper.merge(globalConfig, e.getAsJsonObject());
}
}
}
return InputConfigGson.gson.fromJson(shipperConfigJson, InputConfigImpl.class);
}
-
- private void merge(JsonObject source, JsonObject target) {
- for (Map.Entry<String, JsonElement> e : source.entrySet()) {
- if (!target.has(e.getKey())) {
- target.add(e.getKey(), e.getValue());
- } else {
- if (e.getValue().isJsonObject()) {
- JsonObject valueJson = (JsonObject)e.getValue();
- merge(valueJson, target.get(e.getKey()).getAsJsonObject());
- }
- }
- }
- }
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index a988840..10e38f9 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -92,4 +92,15 @@ public class LogFeederConstants {
public static final String DOCKER_CONTAINER_REGISTRY_ENABLED_PROPERTY = "logfeeder.docker.registry.enabled";
public static final boolean DOCKER_CONTAINER_REGISTRY_ENABLED_DEFAULT = false;
+ public static final String USE_LOCAL_CONFIGS_PROPERTY = "logfeeder.configs.local.enabled";
+ public static final boolean USE_LOCAL_CONFIGS_DEFAULT = false;
+
+ public static final String USE_SOLR_FILTER_STORAGE_PROPERTY = "logfeeder.configs.filter.solr.enabled";
+ public static final boolean USE_SOLR_FILTER_STORAGE_DEFAULT = false;
+
+ public static final String MONITOR_SOLR_FILTER_STORAGE_PROPERTY = "logfeeder.configs.filter.solr.monitor.enabled";
+ public static final boolean MONITOR_SOLR_FILTER_STORAGE_DEFAULT = true;
+
+ public static final String SOLR_ZK_CONNECTION_STRING = "logfeeder.solr.zk_connect_string";
+
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java
index ee8cdcb..b431464 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java
@@ -19,10 +19,8 @@
package org.apache.ambari.logfeeder.conf;
import com.google.common.collect.Maps;
-import org.apache.ambari.logfeeder.ContainerRegistry;
import org.apache.ambari.logfeeder.docker.DockerContainerRegistry;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.docker.DockerContainerRegistryMonitor;
import org.apache.ambari.logfeeder.input.InputConfigUploader;
import org.apache.ambari.logfeeder.input.InputManagerImpl;
import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
@@ -32,9 +30,17 @@ import org.apache.ambari.logfeeder.metrics.StatsLogger;
import org.apache.ambari.logfeeder.output.OutputManagerImpl;
import org.apache.ambari.logfeeder.plugin.manager.InputManager;
import org.apache.ambari.logfeeder.plugin.manager.OutputManager;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterManager;
+import org.apache.ambari.logsearch.config.api.LogLevelFilterUpdater;
import org.apache.ambari.logsearch.config.api.LogSearchConfigFactory;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
+import org.apache.ambari.logsearch.config.local.LogSearchConfigLogFeederLocal;
+import org.apache.ambari.logsearch.config.solr.LogLevelFilterManagerSolr;
+import org.apache.ambari.logsearch.config.solr.LogLevelFilterUpdaterSolr;
import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigLogFeederZK;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
@@ -42,7 +48,6 @@ import org.springframework.context.annotation.PropertySource;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import javax.inject.Inject;
-import java.util.Properties;
@Configuration
@PropertySource(value = {
@@ -72,13 +77,52 @@ public class ApplicationConfig {
@Bean
@DependsOn("logFeederSecurityConfig")
public LogSearchConfigLogFeeder logSearchConfigLogFeeder() throws Exception {
- return LogSearchConfigFactory.createLogSearchConfigLogFeeder(
- Maps.fromProperties(logFeederProps.getProperties()),
- logFeederProps.getClusterName(),
- LogSearchConfigLogFeederZK.class,false);
+ if (logFeederProps.isUseLocalConfigs()) {
+ LogSearchConfigLogFeeder logfeederConfig = LogSearchConfigFactory.createLogSearchConfigLogFeeder(
+ Maps.fromProperties(logFeederProps.getProperties()),
+ logFeederProps.getClusterName(),
+ LogSearchConfigLogFeederLocal.class, false);
+ logfeederConfig.setLogLevelFilterManager(logLevelFilterManager());
+ return logfeederConfig;
+ } else {
+ return LogSearchConfigFactory.createLogSearchConfigLogFeeder(
+ Maps.fromProperties(logFeederProps.getProperties()),
+ logFeederProps.getClusterName(),
+ LogSearchConfigLogFeederZK.class, false);
+ }
}
@Bean
+ public LogLevelFilterManager logLevelFilterManager() {
+ if (logFeederProps.isSolrFilterStorage()) {
+ if (StringUtils.isNotEmpty(logFeederProps.getSolrZkConnectString())) {
+ CloudSolrClient.Builder builder = new CloudSolrClient.Builder();
+ builder.withZkHost(logFeederProps.getSolrZkConnectString());
+ CloudSolrClient solrClient = builder.build();
+ solrClient.setDefaultCollection("history");
+ return new LogLevelFilterManagerSolr(solrClient);
+ } else {
+ return null; // TODO: use lb http client
+ }
+ } else { // no default filter manager
+ return null;
+ }
+ }
+
+ @Bean
+ @DependsOn("logLevelFilterHandler")
+ public LogLevelFilterUpdater logLevelFilterUpdater() throws Exception {
+ if (logFeederProps.isSolrFilterStorage() && logFeederProps.isSolrFilterMonitor()) {
+ LogLevelFilterUpdater logLevelFilterUpdater = new LogLevelFilterUpdaterSolr(
+ "filter-updater-solr", logLevelFilterHandler(),
+ 30, (LogLevelFilterManagerSolr) logLevelFilterManager(), logFeederProps.getClusterName());
+ logLevelFilterUpdater.start();
+ return logLevelFilterUpdater;
+ } else { // no default filter updater
+ return null;
+ }
+ }
+ @Bean
public MetricsManager metricsManager() {
return new MetricsManager();
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index 9a29f86..8f73e2b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -132,7 +132,7 @@ public class LogFeederProps implements LogFeederProperties {
@LogSearchPropertyDescription(
name = LogFeederConstants.DOCKER_CONTAINER_REGISTRY_ENABLED_PROPERTY,
- description = "",
+ description = "Enable to monitor docker containers and store their metadata in an in-memory registry.",
examples = {"true"},
defaultValue = LogFeederConstants.DOCKER_CONTAINER_REGISTRY_ENABLED_DEFAULT + "",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
@@ -140,6 +140,45 @@ public class LogFeederProps implements LogFeederProperties {
@Value("${" + LogFeederConstants.DOCKER_CONTAINER_REGISTRY_ENABLED_PROPERTY + ":false}")
public boolean dockerContainerRegistryEnabled;
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.USE_LOCAL_CONFIGS_PROPERTY,
+ description = "Monitor local input.config-*.json files (do not upload them to zookeeper or solr)",
+ examples = {"true"},
+ defaultValue = LogFeederConstants.USE_LOCAL_CONFIGS_DEFAULT + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.USE_LOCAL_CONFIGS_PROPERTY + ":" + LogFeederConstants.USE_LOCAL_CONFIGS_DEFAULT +"}")
+ public boolean useLocalConfigs;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.USE_SOLR_FILTER_STORAGE_PROPERTY,
+ description = "Use solr as a log level filter storage",
+ examples = {"true"},
+ defaultValue = LogFeederConstants.USE_SOLR_FILTER_STORAGE_DEFAULT + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.USE_SOLR_FILTER_STORAGE_PROPERTY + ":" + LogFeederConstants.USE_SOLR_FILTER_STORAGE_DEFAULT +"}")
+ public boolean solrFilterStorage;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.MONITOR_SOLR_FILTER_STORAGE_PROPERTY,
+ description = "Monitor log level filters (in solr) periodically - used for checking updates.",
+ examples = {"false"},
+ defaultValue = LogFeederConstants.MONITOR_SOLR_FILTER_STORAGE_DEFAULT + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.MONITOR_SOLR_FILTER_STORAGE_PROPERTY + ":" + LogFeederConstants.MONITOR_SOLR_FILTER_STORAGE_DEFAULT +"}")
+ public boolean solrFilterMonitor;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SOLR_ZK_CONNECTION_STRING,
+ description = "Zookeeper connection string for Solr.",
+ examples = {"localhost1:2181,localhost2:2181/mysolr_znode"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SOLR_ZK_CONNECTION_STRING + ":}")
+ private String solrZkConnectString;
+
@Inject
private LogEntryCacheConfig logEntryCacheConfig;
@@ -245,6 +284,38 @@ public class LogFeederProps implements LogFeederProperties {
this.dockerContainerRegistryEnabled = dockerContainerRegistryEnabled;
}
+ public boolean isUseLocalConfigs() {
+ return useLocalConfigs;
+ }
+
+ public void setUseLocalConfigs(boolean useLocalConfigs) {
+ this.useLocalConfigs = useLocalConfigs;
+ }
+
+ public boolean isSolrFilterStorage() {
+ return solrFilterStorage;
+ }
+
+ public void setSolrFilterStorage(boolean solrFilterStorage) {
+ this.solrFilterStorage = solrFilterStorage;
+ }
+
+ public String getSolrZkConnectString() {
+ return solrZkConnectString;
+ }
+
+ public void setSolrZkConnectString(String solrZkConnectString) {
+ this.solrZkConnectString = solrZkConnectString;
+ }
+
+ public boolean isSolrFilterMonitor() {
+ return solrFilterMonitor;
+ }
+
+ public void setSolrFilterMonitor(boolean solrFilterMonitor) {
+ this.solrFilterMonitor = solrFilterMonitor;
+ }
+
@PostConstruct
public void init() {
properties = new Properties();
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
index 3ea3d90..7c633f0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
@@ -26,8 +26,8 @@ import org.apache.ambari.logfeeder.plugin.filter.Filter;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.FilterJsonDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputDescriptorImpl;
import org.apache.commons.collections.MapUtils;
import org.apache.solr.common.util.Base64;
import org.slf4j.Logger;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
index 4eeb197..977e537 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
@@ -36,10 +36,11 @@ import javax.inject.Inject;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
public class LogLevelFilterHandler implements LogLevelFilterMonitor {
private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterHandler.class);
@@ -61,7 +62,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
private LogFeederProps logFeederProps;
private LogSearchConfig config;
- private Map<String, LogLevelFilter> filters = new HashMap<>();
+ private Map<String, LogLevelFilter> filters = new ConcurrentHashMap<>();
public LogLevelFilterHandler(LogSearchConfig config) {
this.config = config;
@@ -70,6 +71,12 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
@PostConstruct
public void init() {
TimeZone.setDefault(TimeZone.getTimeZone(TIMEZONE));
+ if (config.getLogLevelFilterManager() != null) {
+ TreeMap<String, LogLevelFilter> sortedFilters = config.getLogLevelFilterManager()
+ .getLogLevelFilters(logFeederProps.getClusterName())
+ .getFilter();
+ filters = new ConcurrentHashMap<>(sortedFilters);
+ }
}
@Override
@@ -86,6 +93,11 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
}
}
+ @Override
+ public Map<String, LogLevelFilter> getLogLevelFilters() {
+ return filters;
+ }
+
public boolean isAllowed(String hostName, String logId, String level) {
if (!logFeederProps.isLogLevelFilterEnabled()) {
return true;
@@ -144,7 +156,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
defaultFilter.setDefaultLevels(logFeederProps.getIncludeDefaultLogLevels());
try {
- config.createLogLevelFilter(logFeederProps.getClusterName(), logId, defaultFilter);
+ config.getLogLevelFilterManager().createLogLevelFilter(logFeederProps.getClusterName(), logId, defaultFilter);
filters.put(logId, defaultFilter);
} catch (Exception e) {
LOG.warn("Could not persist the default filter for log " + logId, e);
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
index dafe6a5..2113cbd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
@@ -21,7 +21,6 @@ package org.apache.ambari.logfeeder.output;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.output.Output;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
import org.apache.log4j.Logger;
import java.io.File;
@@ -54,10 +53,6 @@ public class OutputDevNull extends Output<LogFeederProps, InputMarker> {
}
@Override
- public void outputConfigChanged(OutputProperties outputProperties) {
- }
-
- @Override
public void close() {
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
index e70d769..850daaf 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
@@ -24,7 +24,6 @@ import org.apache.ambari.logfeeder.input.InputFileMarker;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.lang3.StringUtils;
@@ -156,10 +155,6 @@ public class OutputFile extends Output<LogFeederProps, InputFileMarker> {
}
@Override
- public void outputConfigChanged(OutputProperties outputProperties) {
- }
-
- @Override
public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
throw new UnsupportedOperationException("copyFile method is not yet supported for output=file");
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
index f2faf64..03669fe 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
@@ -30,7 +30,6 @@ import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.LogFeederHDFSUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.PlaceholderUtil;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
@@ -246,10 +245,6 @@ public class OutputHDFSFile extends Output<LogFeederProps, InputFileMarker> impl
}
@Override
- public void outputConfigChanged(OutputProperties outputProperties) {
- }
-
- @Override
public Long getPendingCount() {
return 0L;
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
index 7539484..a82ede0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
@@ -24,7 +24,6 @@ import org.apache.ambari.logfeeder.input.InputFileMarker;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
@@ -293,11 +292,6 @@ public class OutputKafka extends Output<LogFeederProps, InputFileMarker> {
}
@Override
- public void outputConfigChanged(OutputProperties outputProperties) {
- throw new IllegalStateException("This method should be overriden if the Output wants to monitor the configuration");
- };
-
- @Override
public Long getPendingCount() {
return 0L;
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
index bae7eec..fc64d4b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
@@ -33,10 +33,10 @@ import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.S3Util;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputS3FileDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigGson;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputConfigImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputS3FileDescriptorImpl;
import org.apache.log4j.Logger;
import java.io.File;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index 6330dae..041c1bd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -24,8 +24,6 @@ import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.DateUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -36,8 +34,9 @@ import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.CollectionStateWatcher;
import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
import java.io.File;
import java.io.IOException;
@@ -54,12 +53,12 @@ import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
-public class OutputSolr extends Output<LogFeederProps, InputMarker> implements CollectionStateWatcher {
+public class OutputSolr extends Output<LogFeederProps, InputMarker> {
private static final Logger LOG = Logger.getLogger(OutputSolr.class);
- private static final int OUTPUT_PROPERTIES_WAIT_MS = 10000;
private static final int SHARDS_WAIT_MS = 10000;
private static final int DEFAULT_MAX_BUFFER_SIZE = 5000;
@@ -76,7 +75,6 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
private String collection;
private String splitMode;
private int splitInterval;
- private List<String> shards;
private String zkConnectString;
private int maxIntervalMS;
private int workers;
@@ -85,8 +83,6 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
private int lastSlotByMin = -1;
private boolean skipLogtime = false;
- private final Object propertiesLock = new Object();
-
private BlockingQueue<OutputData> outgoingBuffer = null;
private List<SolrWorkerThread> workerThreadList = new ArrayList<>();
@@ -118,22 +114,11 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
initParams(logFeederProps);
setupSecurity();
createOutgoingBuffer();
- createSolrStateWatcher();
createSolrWorkers();
}
private void initParams(LogFeederProps logFeederProps) throws Exception {
type = getStringValue("type");
- while (true) {
- OutputSolrProperties outputSolrProperties = getLogSearchConfig().getOutputSolrProperties(type);
- if (outputSolrProperties == null) {
- LOG.info("Output solr properties for type " + type + " is not available yet.");
- try { Thread.sleep(OUTPUT_PROPERTIES_WAIT_MS); } catch (Exception e) { LOG.warn(e); }
- } else {
- initPropertiesFromLogSearchConfig(outputSolrProperties, true);
- break;
- }
- }
zkConnectString = getStringValue("zk_connect_string");
if (StringUtils.isEmpty(zkConnectString)) {
@@ -145,6 +130,17 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
maxIntervalMS = getIntValue("idle_flush_time_ms", DEFAULT_MAX_INTERVAL_MS);
workers = getIntValue("workers", DEFAULT_NUMBER_OF_WORKERS);
+ splitInterval = 0;
+ splitMode = getStringValue("split_interval", "none");
+ if (!splitMode.equals("none")) {
+ splitInterval = Integer.parseInt(splitMode);
+ }
+
+ collection = getStringValue("collection");
+ if (StringUtils.isEmpty(collection)) {
+ throw new IllegalStateException("Collection property is mandatory");
+ }
+
maxBufferSize = getIntValue("flush_size", DEFAULT_MAX_BUFFER_SIZE);
if (maxBufferSize < 1) {
LOG.warn("maxBufferSize is less than 1. Making it 1");
@@ -162,28 +158,6 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
}
}
- @Override
- public void outputConfigChanged(OutputProperties outputProperties) {
- initPropertiesFromLogSearchConfig((OutputSolrProperties)outputProperties, false);
- }
-
- private void initPropertiesFromLogSearchConfig(OutputSolrProperties outputSolrProperties, boolean init) {
- synchronized (propertiesLock) {
- splitMode = outputSolrProperties.getSplitIntervalMins();
- if (!splitMode.equalsIgnoreCase("none")) {
- splitInterval = Integer.parseInt(splitMode);
- }
-
- // collection can not be overwritten after initialization
- if (init) {
- collection = outputSolrProperties.getCollection();
- if (StringUtils.isEmpty(collection)) {
- throw new IllegalStateException("Collection property is mandatory");
- }
- }
- }
- }
-
private void setupSecurity() {
boolean securityEnabled = logFeederProps.getLogFeederSecurityConfig().isSolrKerberosEnabled();
if (securityEnabled) {
@@ -200,35 +174,6 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
outgoingBuffer = new LinkedBlockingQueue<OutputData>(bufferSize);
}
- private void createSolrStateWatcher() throws Exception {
- if ("none".equals(splitMode)) {
- return;
- }
-
- CloudSolrClient stateWatcherClient = createSolrClient();
- stateWatcherClient.registerCollectionStateWatcher(collection, this);
- while (true) {
- if (shards == null) {
- LOG.info("Shards are not available yet, waiting ...");
- try { Thread.sleep(SHARDS_WAIT_MS); } catch (Exception e) { LOG.warn(e); }
- } else {
- break;
- }
- }
- }
-
- @Override
- public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
- synchronized (propertiesLock) {
- if (collectionState != null) {
- List<String> shards = new ArrayList<>(collectionState.getSlicesMap().keySet());
- Collections.sort(shards);
- this.shards = shards;
- }
- }
- return false;
- }
-
private void createSolrWorkers() throws Exception, MalformedURLException {
for (int count = 0; count < workers; count++) {
CloudSolrClient solrClient = getSolrClient(count);
@@ -429,11 +374,9 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
boolean result = false;
while (!isDrain()) {
try {
- synchronized (propertiesLock) {
- if (implicitRouting) {
- // Compute the current router value
- addRouterField();
- }
+ if (implicitRouting) {
+ // Compute the current router value
+ addRouterField();
}
addToSolr(outputData);
resetLocalBuffer();
@@ -492,6 +435,11 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
}
private void addRouterField() {
+ ZkStateReader reader = ((CloudSolrClient) solrClient).getZkStateReader();
+ DocCollection docCollection = reader.getClusterState().getCollection(collection);
+ Collection<Slice> slices = docCollection.getSlices();
+ List<String> shards = slices.stream().map(Slice::getName).collect(Collectors.toList());
+
Calendar cal = Calendar.getInstance();
int weekDay = cal.get(Calendar.DAY_OF_WEEK);
int currHour = cal.get(Calendar.HOUR_OF_DAY);
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json
index 94b44da..4f1ad6d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json
@@ -1,8 +1,9 @@
{
"output": [
{
- "is_enabled": "true",
"comment": "Output to solr for service logs",
+ "is_enabled": "true",
+ "collection": "hadoop_logs",
"destination": "solr",
"zk_connect_string": "localhost:2181",
"type": "service",
@@ -18,6 +19,7 @@
{
"comment": "Output to solr for audit records",
"is_enabled": "true",
+ "collection": "audit_logs",
"destination": "solr",
"zk_connect_string": "localhost:2181",
"type": "audit",
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 5217309..0fb1058 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -35,3 +35,6 @@ logfeeder.cache.last.dedup.enabled=true
#logfeeder tmp dir
logfeeder.tmp.dir=${LOGFEEDER_RELATIVE_LOCATION:}target/tmp
+
+#logfeeder.configs.local.enabled=true
+#logfeeder.configs.filter.solr.enabled=true
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
index 49fb301..8d3967b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
@@ -25,7 +25,7 @@ import org.apache.ambari.logfeeder.input.InputFileMarker;
import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.manager.OutputManager;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterGrokDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.FilterGrokDescriptorImpl;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 36139ea..a328eb8 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -29,7 +29,7 @@ import org.apache.ambari.logfeeder.common.LogFeederException;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputFileMarker;
import org.apache.ambari.logfeeder.plugin.manager.OutputManager;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.FilterJsonDescriptorImpl;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
index ff8cb6e..efda7e2 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
@@ -24,7 +24,7 @@ import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputFileMarker;
import org.apache.ambari.logfeeder.plugin.manager.OutputManager;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterKeyValueDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.FilterKeyValueDescriptorImpl;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
index e3349fc..2219be1 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
@@ -29,7 +29,7 @@ import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.plugin.filter.Filter;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.manager.InputManager;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputFileDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputFileDescriptorImpl;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.easymock.EasyMock;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperAnonymizeTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperAnonymizeTest.java
index b6a527d..c22f312 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperAnonymizeTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperAnonymizeTest.java
@@ -21,7 +21,7 @@ package org.apache.ambari.logfeeder.mapper;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapAnonymizeDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.MapAnonymizeDescriptorImpl;
import org.apache.log4j.Logger;
import org.junit.Test;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
index 0a0a9fd..5e94996 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
@@ -25,7 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapDateDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.MapDateDescriptorImpl;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.log4j.Logger;
import org.junit.Test;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java
index 4899dfc..5c6cc93 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java
@@ -21,7 +21,7 @@ package org.apache.ambari.logfeeder.mapper;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapFieldCopyDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.MapFieldCopyDescriptorImpl;
import org.apache.log4j.Logger;
import org.junit.Test;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
index 74b88fc..f74c9f8 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
@@ -21,7 +21,7 @@ package org.apache.ambari.logfeeder.mapper;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapFieldNameDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.MapFieldNameDescriptorImpl;
import org.apache.log4j.Logger;
import org.junit.Test;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
index 1a33740..92befa9 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
@@ -21,7 +21,7 @@ package org.apache.ambari.logfeeder.mapper;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapFieldValueDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.MapFieldValueDescriptorImpl;
import org.apache.log4j.Logger;
import org.junit.Test;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java
index e97ccaf..502641f 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java
@@ -22,7 +22,7 @@ import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.input.cache.LRUCache;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputDescriptorImpl;
import org.easymock.EasyMock;
import org.junit.Before;
import org.junit.Test;
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
index f1c27ab..1a651ab 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
@@ -35,7 +35,7 @@ import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
import org.apache.ambari.logfeeder.plugin.common.MetricData;
import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.output.Output;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
+import org.apache.ambari.logsearch.config.json.model.inputconfig.impl.InputDescriptorImpl;
import org.junit.Test;
public class OutputManagerTest {
diff --git a/ambari-logsearch/ambari-logsearch-server/pom.xml b/ambari-logsearch/ambari-logsearch-server/pom.xml
index 7455043..32927fe 100755
--- a/ambari-logsearch/ambari-logsearch-server/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-server/pom.xml
@@ -353,6 +353,11 @@
</dependency>
<dependency>
<groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-config-solr</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
<artifactId>ambari-logsearch-config-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
index 228f1c6..4683df4 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
@@ -35,6 +35,7 @@ public enum MessageEnums {
SOLR_COLLECTION_NOT_READY("logsearch.solr.collection.error", "Solr has not accessible yet for collection."),
CONFIGURATION_NOT_AVAILABLE("logsearch.config.not_available", "Log Search configuration is not available"),
CONFIGURATION_API_DISABLED("logsearch.config.api.disabled", "Log Search configuration is not available"),
+ SOLR_CONFIGURATION_API_SOLR_NOT_AVAILEBLE("logsearch.config.api.solr.not.available", "Solr as log level filter manager source is not available"),
// Common Validations
INVALID_PASSWORD("logsearch.validation.invalid_password", "Invalid password"),
INVALID_INPUT_DATA("logsearch.validation.invalid_input_data", "Invalid input data"),
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
index 7d4bc2c..accf11a 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
@@ -33,7 +33,7 @@ import java.util.Set;
@Named
public class ShipperConfigDescriptionStorage {
- private static final String SHIPPER_CONFIG_PACKAGE = "org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl";
+ private static final String SHIPPER_CONFIG_PACKAGE = "org.apache.ambari.logsearch.config.json.model.inputconfig.impl";
private final List<ShipperConfigDescriptionData> shipperConfigDescription = new ArrayList<>();
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
index d957349..b7259d6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
@@ -59,5 +59,4 @@ public class ApplicationConfig {
factoryBean.afterPropertiesSet();
return factoryBean.getObject();
}
-
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java
index 6e16179..7f3632d 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java
@@ -27,7 +27,6 @@ import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_PR
@Configuration
public class LogSearchConfigApiConfig {
- @Value("${logsearch.config.api.enabled:true}")
@LogSearchPropertyDescription(
name = "logsearch.config.api.enabled",
description = "Enable config API feature and shipperconfig API endpoints.",
@@ -35,8 +34,19 @@ public class LogSearchConfigApiConfig {
defaultValue = "true",
sources = {LOGSEARCH_PROPERTIES_FILE}
)
+ @Value("${logsearch.config.api.enabled:true}")
private boolean configApiEnabled;
+ @LogSearchPropertyDescription(
+ name = "logsearch.config.api.filter.solr.enabled",
+ description = "Use solr as a log level filter storage",
+ examples = {"true"},
+ defaultValue = "false",
+ sources = {LOGSEARCH_PROPERTIES_FILE}
+ )
+ @Value("${logsearch.config.api.filter.solr.enabled:false}")
+ public boolean solrFilterStorage;
+
public boolean isConfigApiEnabled() {
return configApiEnabled;
}
@@ -44,4 +54,12 @@ public class LogSearchConfigApiConfig {
public void setConfigApiEnabled(boolean configApiEnabled) {
this.configApiEnabled = configApiEnabled;
}
+
+ public boolean isSolrFilterStorage() {
+ return this.solrFilterStorage;
+ }
+
+ public void setSolrFilterStorage(boolean solrFilterStorage) {
+ this.solrFilterStorage = solrFilterStorage;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
index 42d5b66..69d63d2 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
@@ -22,10 +22,12 @@ import com.google.common.collect.Lists;
import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.conf.global.SolrLogLevelFilterManagerState;
import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler;
import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler;
import org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler;
+import org.apache.ambari.logsearch.web.filters.LogSearchLogLevelFilterManagerFilter;
import org.apache.ambari.logsearch.web.filters.LogsearchAuditLogsStateFilter;
import org.apache.ambari.logsearch.web.filters.LogsearchAuthenticationEntryPoint;
import org.apache.ambari.logsearch.web.filters.LogsearchCorsFilter;
@@ -88,6 +90,10 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
private SolrCollectionState solrEventHistoryState;
@Inject
+ @Named("solrLogLevelFilterManagerState")
+ private SolrLogLevelFilterManagerState solrLogLevelFilterManagerState;
+
+ @Inject
private LogSearchConfigState logSearchConfigState;
@Inject
@@ -112,6 +118,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
.addFilterAfter(logsearchAuditLogFilter(), LogsearchSecurityContextFormationFilter.class)
.addFilterAfter(logsearchServiceLogFilter(), LogsearchSecurityContextFormationFilter.class)
.addFilterAfter(logSearchConfigStateFilter(), LogsearchSecurityContextFormationFilter.class)
+ .addFilterAfter(logSearchLogLevelFilterManagerFilter(), LogsearchSecurityContextFormationFilter.class)
.addFilterBefore(logsearchCorsFilter(), LogsearchSecurityContextFormationFilter.class)
.addFilterBefore(logsearchJwtFilter(), LogsearchSecurityContextFormationFilter.class)
.logout()
@@ -183,7 +190,17 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Bean
public LogSearchConfigStateFilter logSearchConfigStateFilter() {
- return new LogSearchConfigStateFilter(logsearchConfigRequestMatcher(), logSearchConfigState, logSearchConfigApiConfig.isConfigApiEnabled());
+ if (logSearchConfigApiConfig.isSolrFilterStorage()) {
+ return new LogSearchConfigStateFilter(shipperConfigInputRequestMatcher(), logSearchConfigState, logSearchConfigApiConfig.isConfigApiEnabled());
+ } else {
+ return new LogSearchConfigStateFilter(logsearchConfigRequestMatcher(), logSearchConfigState, logSearchConfigApiConfig.isConfigApiEnabled());
+ }
+ }
+
+ @Bean
+ public LogSearchLogLevelFilterManagerFilter logSearchLogLevelFilterManagerFilter() {
+ boolean enabled = logSearchConfigApiConfig.isSolrFilterStorage() && !logSearchConfigApiConfig.isConfigApiEnabled();
+ return new LogSearchLogLevelFilterManagerFilter(logLevelFilterRequestMatcher(), solrLogLevelFilterManagerState, enabled);
}
@Bean
@@ -224,6 +241,14 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
return new AntPathRequestMatcher("/api/v1/shipper/**");
}
+ public RequestMatcher logLevelFilterRequestMatcher() {
+ return new AntPathRequestMatcher("/api/v1/shipper/filters/**");
+ }
+
+ public RequestMatcher shipperConfigInputRequestMatcher() {
+ return new AntPathRequestMatcher("/api/v1/shipper/input/**");
+ }
+
private String[] getCookies() {
List<String> cookies = new ArrayList<>();
cookies.add(LOGSEARCH_SESSION_ID);
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/outputconfig/impl/OutputSolrPropertiesImpl.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrClientsHolder.java
similarity index 52%
rename from ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/outputconfig/impl/OutputSolrPropertiesImpl.java
rename to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrClientsHolder.java
index 4b9f54c..913472f 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/outputconfig/impl/OutputSolrPropertiesImpl.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrClientsHolder.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,31 +16,34 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.ambari.logsearch.conf;
-package org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl;
+import org.apache.solr.client.solrj.SolrClient;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
+import java.util.EnumMap;
+import java.util.Map;
-import com.google.gson.annotations.SerializedName;
+public class SolrClientsHolder {
-public class OutputSolrPropertiesImpl implements OutputSolrProperties {
- private final String collection;
+ private Map<CollectionType, SolrClient> clientsMap = new EnumMap<>(CollectionType.class);
- @SerializedName("split_interval_mins")
- private final String splitIntervalMins;
+ public enum CollectionType {
+ SERVICE,
+ AUDIT,
+ HISTORY
+ }
- public OutputSolrPropertiesImpl(String collection, String splitIntervalMins) {
- this.collection = collection;
- this.splitIntervalMins = splitIntervalMins;
+ public SolrClientsHolder() {
+ clientsMap.put(CollectionType.SERVICE, null);
+ clientsMap.put(CollectionType.AUDIT, null);
+ clientsMap.put(CollectionType.HISTORY, null);
}
- @Override
- public String getCollection() {
- return collection;
+ public SolrClient getSolrClient(CollectionType type) {
+ return clientsMap.get(type);
}
- @Override
- public String getSplitIntervalMins() {
- return splitIntervalMins;
+ public synchronized void setSolrClient(SolrClient solrClient, CollectionType type) {
+ clientsMap.put(type, solrClient);
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
index f9144ff..7577d6e 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
@@ -20,13 +20,12 @@ package org.apache.ambari.logsearch.conf;
import org.apache.ambari.logsearch.conf.global.SolrAuditLogsState;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.conf.global.SolrLogLevelFilterManagerState;
import org.apache.ambari.logsearch.conf.global.SolrServiceLogsState;
import org.apache.ambari.logsearch.conf.global.SolrEventHistoryState;
import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.DependsOn;
-import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.repository.config.EnableSolrRepositories;
import org.springframework.scheduling.annotation.EnableScheduling;
@@ -35,25 +34,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
public class SolrConfig {
- @Bean(name = "serviceSolrTemplate")
- public SolrTemplate serviceSolrTemplate() {
- return null;
- }
-
- @Bean(name = "auditSolrTemplate")
- @DependsOn("serviceSolrTemplate")
- public SolrTemplate auditSolrTemplate() {
- return null;
- }
-
- @Bean(name = "eventHistorySolrTemplate")
- @DependsOn("serviceSolrTemplate")
- public SolrTemplate eventHistorySolrTemplate() {
- return null;
- }
-
@Bean
- @DependsOn({"serviceSolrTemplate", "auditSolrTemplate"})
public SolrSchemaFieldDao solrSchemaFieldDao() {
return new SolrSchemaFieldDao();
}
@@ -72,5 +53,15 @@ public class SolrConfig {
public SolrCollectionState solrEventHistoryState() {
return new SolrEventHistoryState();
}
+
+ @Bean(name = "solrLogLevelFilterManagerState")
+ public SolrLogLevelFilterManagerState solrLogLevelFilterManagerState() {
+ return new SolrLogLevelFilterManagerState();
+ }
+
+ @Bean
+ public SolrClientsHolder solrClientsHolder() {
+ return new SolrClientsHolder();
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/outputconfig/OutputSolrProperties.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/global/SolrLogLevelFilterManagerState.java
similarity index 64%
rename from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/outputconfig/OutputSolrProperties.java
rename to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/global/SolrLogLevelFilterManagerState.java
index 586e785..d610e10 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/outputconfig/OutputSolrProperties.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/global/SolrLogLevelFilterManagerState.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,11 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.ambari.logsearch.conf.global;
-package org.apache.ambari.logsearch.config.api.model.outputconfig;
+import javax.inject.Named;
-public interface OutputSolrProperties extends OutputProperties {
- String getCollection();
+@Named
+public class SolrLogLevelFilterManagerState {
- String getSplitIntervalMins();
+ private volatile boolean logLevelFilterManagerIsReady;
+
+ public boolean isLogLevelFilterManagerIsReady() {
+ return logLevelFilterManagerIsReady;
+ }
+
+ public void setLogLevelFilterManagerIsReady(boolean logLevelFilterManagerIsReady) {
+ this.logLevelFilterManagerIsReady = logLevelFilterManagerIsReady;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
index 67f7363..9625736 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.configurer;
+import org.apache.ambari.logsearch.conf.SolrClientsHolder;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
import org.apache.ambari.logsearch.dao.SolrDaoBase;
@@ -27,8 +28,8 @@ import org.apache.ambari.logsearch.handler.ListCollectionHandler;
import org.apache.ambari.logsearch.handler.ReloadCollectionHandler;
import org.apache.ambari.logsearch.handler.UploadConfigurationHandler;
import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
@@ -55,10 +56,15 @@ public class SolrCollectionConfigurer implements Configurer {
private final SolrDaoBase solrDaoBase;
private final boolean hasEnumConfig; // enumConfig.xml for solr collection
+ private final SolrClientsHolder solrClientsHolder;
+ private final SolrClientsHolder.CollectionType collectionType;
- public SolrCollectionConfigurer(SolrDaoBase solrDaoBase, boolean hasEnumConfig) {
+ public SolrCollectionConfigurer(SolrDaoBase solrDaoBase, boolean hasEnumConfig,
+ SolrClientsHolder solrClientsHolder, SolrClientsHolder.CollectionType collectionType) {
this.solrDaoBase = solrDaoBase;
this.hasEnumConfig = hasEnumConfig;
+ this.solrClientsHolder = solrClientsHolder;
+ this.collectionType = collectionType;
}
@Override
@@ -86,7 +92,7 @@ public class SolrCollectionConfigurer implements Configurer {
if (solrDaoBase.getSolrTemplate() == null) {
solrDaoBase.setSolrTemplate(createSolrTemplate(solrPropsConfig));
}
- CloudSolrClient cloudSolrClient = (CloudSolrClient) solrDaoBase.getSolrTemplate().getSolrClient();
+ CloudSolrClient cloudSolrClient = (CloudSolrClient) solrClientsHolder.getSolrClient(collectionType);
boolean reloadCollectionNeeded = uploadConfigurationsIfNeeded(cloudSolrClient, configSetFolder, state, solrPropsConfig);
checkSolrStatus(cloudSolrClient);
createCollectionsIfNeeded(cloudSolrClient, state, solrPropsConfig, reloadCollectionNeeded);
@@ -114,10 +120,12 @@ public class SolrCollectionConfigurer implements Configurer {
}
public SolrTemplate createSolrTemplate(SolrPropsConfig solrPropsConfig) {
- return new SolrTemplate(createClient(
+ SolrClient solrClient = createClient(
solrPropsConfig.getSolrUrl(),
solrPropsConfig.getZkConnectString(),
- solrPropsConfig.getCollection()));
+ solrPropsConfig.getCollection());
+ solrClientsHolder.setSolrClient(solrClient, collectionType);
+ return new SolrTemplate(solrClient);
}
private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) {
@@ -219,7 +227,7 @@ public class SolrCollectionConfigurer implements Configurer {
try {
List<String> allCollectionList = new ListCollectionHandler().handle(solrClient, null);
solrDaoBase.waitForLogSearchConfig();
- CreateCollectionHandler handler = new CreateCollectionHandler(solrDaoBase.getLogSearchConfig(), allCollectionList);
+ CreateCollectionHandler handler = new CreateCollectionHandler(allCollectionList);
boolean collectionCreated = handler.handle(solrClient, solrPropsConfig);
boolean collectionReloaded = true;
if (reloadCollectionNeeded) {
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrLogLevelFilterConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrLogLevelFilterConfigurer.java
new file mode 100644
index 0000000..e62c6c4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrLogLevelFilterConfigurer.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.configurer;
+
+import org.apache.ambari.logsearch.conf.LogSearchConfigApiConfig;
+import org.apache.ambari.logsearch.conf.global.SolrLogLevelFilterManagerState;
+import org.apache.ambari.logsearch.config.solr.LogLevelFilterManagerSolr;
+import org.apache.ambari.logsearch.dao.EventHistorySolrDao;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import javax.inject.Named;
+
+@Named
+public class SolrLogLevelFilterConfigurer implements Configurer {
+ private static final Logger logger = LoggerFactory.getLogger(SolrLogLevelFilterConfigurer.class);
+
+ private static final int RETRY_INTERVAL_SECONDS = 10;
+
+ private final EventHistorySolrDao eventHistorySolrDao;
+ private final SolrLogLevelFilterManagerState solrLogLevelFilterManagerState;
+ private final LogSearchConfigApiConfig logSearchConfigApiConfig;
+
+ private LogLevelFilterManagerSolr logLevelFilterManagerSolr;
+
+ @Inject
+ public SolrLogLevelFilterConfigurer(final LogSearchConfigApiConfig logSearchConfigApiConfig,
+ final SolrLogLevelFilterManagerState solrLogLevelFilterManagerState,
+ final EventHistorySolrDao eventHistorySolrDao) {
+ this.logSearchConfigApiConfig = logSearchConfigApiConfig;
+ this.solrLogLevelFilterManagerState = solrLogLevelFilterManagerState;
+ this.eventHistorySolrDao = eventHistorySolrDao;
+ }
+
+ @PostConstruct
+ @Override
+ public void start() {
+ Thread setupThread = new Thread("setup_solr_loglevel_filter_manager") {
+ @Override
+ public void run() {
+ logger.info("Start initializing log level filter manager ...");
+ if (logSearchConfigApiConfig.isSolrFilterStorage()) {
+ while (true) {
+ try {
+ if (eventHistorySolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+ setLogLevelFilterManagerSolr(new LogLevelFilterManagerSolr(eventHistorySolrDao.getSolrClient()));
+ solrLogLevelFilterManagerState.setLogLevelFilterManagerIsReady(true);
+ logger.info("Log level filter manager successfully initialized.");
+ break;
+ }
+ } catch (Exception ex) {
+ logger.warn("Could not initialize log level Solr filter manager, going to sleep for " + RETRY_INTERVAL_SECONDS + " seconds ", ex);
+ }
+ try {
+ Thread.sleep(RETRY_INTERVAL_SECONDS * 1000);
+ } catch (Exception e) {/* ignore */}
+ }
+ } else {
+ logger.info("Solr is not used as a log level filter storage.");
+ }
+ }
+ };
+ setupThread.setDaemon(true);
+ setupThread.start();
+ }
+
+ public LogLevelFilterManagerSolr getLogLevelFilterManagerSolr() {
+ return logLevelFilterManagerSolr;
+ }
+
+ public void setLogLevelFilterManagerSolr(final LogLevelFilterManagerSolr logLevelFilterManagerSolr) {
+ this.logLevelFilterManagerSolr = logLevelFilterManagerSolr;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
index 3eea08f..4bef51e 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
@@ -25,6 +25,7 @@ import javax.inject.Named;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
+import org.apache.ambari.logsearch.conf.SolrClientsHolder;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
import org.apache.ambari.logsearch.configurer.SolrAuditAliasConfigurer;
@@ -41,14 +42,15 @@ public class AuditSolrDao extends SolrDaoBase {
@Inject
private SolrAuditLogPropsConfig solrAuditLogPropsConfig;
- @Inject
- @Named("auditSolrTemplate")
private SolrTemplate auditSolrTemplate;
@Inject
@Named("solrAuditLogsState")
private SolrCollectionState solrAuditLogsState;
+ @Inject
+ private SolrClientsHolder solrClientsHolder;
+
public AuditSolrDao() {
super(LogType.AUDIT);
}
@@ -69,7 +71,7 @@ public class AuditSolrDao extends SolrDaoBase {
String rangerAuditCollection = solrAuditLogPropsConfig.getRangerCollection();
try {
- new SolrCollectionConfigurer(this, true).start();
+ new SolrCollectionConfigurer(this, true, solrClientsHolder, SolrClientsHolder.CollectionType.AUDIT).start();
boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection));
if (createAlias) {
new SolrAuditAliasConfigurer(this).start();
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/EventHistorySolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/EventHistorySolrDao.java
index aff4061..e85702c 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/EventHistorySolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/EventHistorySolrDao.java
@@ -27,6 +27,7 @@ import javax.inject.Named;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.LogType;
+import org.apache.ambari.logsearch.conf.SolrClientsHolder;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.SolrEventHistoryPropsConfig;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
@@ -49,14 +50,15 @@ public class EventHistorySolrDao extends SolrDaoBase {
@Inject
private SolrEventHistoryPropsConfig solrEventHistoryPropsConfig;
- @Inject
- @Named("eventHistorySolrTemplate")
private SolrTemplate eventHistorySolrTemplate;
@Inject
@Named("solrEventHistoryState")
private SolrCollectionState solrEventHistoryState;
+ @Inject
+ private SolrClientsHolder solrClientsHolder;
+
public EventHistorySolrDao() {
super(LogType.SERVICE);
}
@@ -78,7 +80,7 @@ public class EventHistorySolrDao extends SolrDaoBase {
String collection = solrEventHistoryPropsConfig.getCollection();
try {
- new SolrCollectionConfigurer(this, false).start();
+ new SolrCollectionConfigurer(this, false, solrClientsHolder, SolrClientsHolder.CollectionType.HISTORY).start();
} catch (Exception e) {
LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString +
", collection=" + collection, e);
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
index 308ef1f..4fbe534 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
@@ -24,6 +24,7 @@ import javax.inject.Inject;
import javax.inject.Named;
import org.apache.ambari.logsearch.common.LogType;
+import org.apache.ambari.logsearch.conf.SolrClientsHolder;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
@@ -39,14 +40,15 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
@Inject
private SolrServiceLogPropsConfig solrServiceLogPropsConfig;
- @Inject
- @Named("serviceSolrTemplate")
private volatile SolrTemplate serviceSolrTemplate;
@Inject
@Named("solrServiceLogsState")
private SolrCollectionState solrServiceLogsState;
+ @Inject
+ private SolrClientsHolder solrClientsHolder;
+
public ServiceLogsSolrDao() {
super(LogType.SERVICE);
}
@@ -65,7 +67,7 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
public void postConstructor() {
LOG.info("postConstructor() called.");
try {
- new SolrCollectionConfigurer(this, true).start();
+ new SolrCollectionConfigurer(this, true, solrClientsHolder, SolrClientsHolder.CollectionType.HISTORY).start();
} catch (Exception e) {
LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl()
+ ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString()
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index ba5432f..a13c27f 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -19,9 +19,6 @@
package org.apache.ambari.logsearch.handler;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
-import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
-import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
-import org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl.OutputSolrPropertiesImpl;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
@@ -54,22 +51,14 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
- private final LogSearchConfigServer logSearchConfig;
private final List<String> allCollectionList;
- public CreateCollectionHandler(LogSearchConfigServer logSearchConfig, List<String> allCollectionList) {
- this.logSearchConfig = logSearchConfig;
+ public CreateCollectionHandler(List<String> allCollectionList) {
this.allCollectionList = allCollectionList;
}
@Override
public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
- if (solrPropsConfig.getLogType() != null) {
- OutputSolrProperties outputSolrProperties = new OutputSolrPropertiesImpl(solrPropsConfig.getCollection(),
- solrPropsConfig.getSplitInterval());
- logSearchConfig.saveOutputSolrProperties(solrPropsConfig.getLogType(), outputSolrProperties);
- }
-
boolean result;
if (solrPropsConfig.isSolrImplicitRouting()) {
result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, this.allCollectionList);
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
index 3c667c2..a7a955c 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
@@ -24,8 +24,10 @@ import java.util.Map;
import java.util.Set;
import org.apache.ambari.logfeeder.common.LogEntryParseTester;
+import org.apache.ambari.logsearch.conf.LogSearchConfigApiConfig;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
import org.apache.ambari.logsearch.configurer.LogSearchConfigConfigurer;
+import org.apache.ambari.logsearch.configurer.SolrLogLevelFilterConfigurer;
import org.apache.ambari.logsearch.model.common.LSServerInputConfig;
import org.apache.ambari.logsearch.model.common.LSServerLogLevelFilterMap;
import org.apache.log4j.Logger;
@@ -47,7 +49,13 @@ public class ShipperConfigManager extends JsonManagerBase {
private static final Logger logger = Logger.getLogger(ShipperConfigManager.class);
@Inject
+ private LogSearchConfigApiConfig logSearchConfigApiConfig;
+
+ @Inject
private LogSearchConfigConfigurer logSearchConfigConfigurer;
+
+ @Inject
+ private SolrLogLevelFilterConfigurer solrLogLevelFilterConfigurer;
public List<String> getServices(String clusterName) {
return logSearchConfigConfigurer.getConfig().getServices(clusterName);
@@ -111,12 +119,20 @@ public class ShipperConfigManager extends JsonManagerBase {
}
public LSServerLogLevelFilterMap getLogLevelFilters(String clusterName) {
- return new LSServerLogLevelFilterMap(logSearchConfigConfigurer.getConfig().getLogLevelFilters(clusterName));
+ if (logSearchConfigApiConfig.isSolrFilterStorage()) {
+ return new LSServerLogLevelFilterMap(solrLogLevelFilterConfigurer.getLogLevelFilterManagerSolr().getLogLevelFilters(clusterName));
+ } else {
+ return new LSServerLogLevelFilterMap(logSearchConfigConfigurer.getConfig().getLogLevelFilterManager().getLogLevelFilters(clusterName));
+ }
}
public Response setLogLevelFilters(String clusterName, LSServerLogLevelFilterMap request) {
try {
- logSearchConfigConfigurer.getConfig().setLogLevelFilters(clusterName, request.convertToApi());
+ if (logSearchConfigApiConfig.isSolrFilterStorage()) {
+ solrLogLevelFilterConfigurer.getLogLevelFilterManagerSolr().setLogLevelFilters(clusterName, request.convertToApi());
+ } else {
+ logSearchConfigConfigurer.getConfig().getLogLevelFilterManager().setLogLevelFilters(clusterName, request.convertToApi());
+ }
return Response.ok().build();
} catch (Exception e) {
logger.warn("Could not update log level filters", e);
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchLogLevelFilterManagerFilter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchLogLevelFilterManagerFilter.java
new file mode 100644
index 0000000..d8b2ced
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchLogLevelFilterManagerFilter.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.global.SolrLogLevelFilterManagerState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+public class LogSearchLogLevelFilterManagerFilter implements Filter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogSearchLogLevelFilterManagerFilter.class);
+
+ private static final String SOLR_FILTER_MANAGER_NOT_AVAILABLE = "Solr log level filter manager is not available";
+
+ private final RequestMatcher requestMatcher;
+ private final SolrLogLevelFilterManagerState solrLogLevelFilterManagerState;
+ private final boolean enabled;
+
+ public LogSearchLogLevelFilterManagerFilter(RequestMatcher requestMatcher,
+ SolrLogLevelFilterManagerState solrLogLevelFilterManagerState,
+ boolean enabled) {
+ this.requestMatcher = requestMatcher;
+ this.solrLogLevelFilterManagerState = solrLogLevelFilterManagerState;
+ this.enabled = enabled;
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+ HttpServletRequest request = (HttpServletRequest) servletRequest;
+ if (enabled) {
+ if (requestMatcher.matches(request)) {
+ VResponse errorResponse = getErrorResponse();
+ if (errorResponse != null) {
+ LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc());
+ HttpServletResponse resp = (HttpServletResponse) servletResponse;
+ resp.setStatus(500);
+ resp.setContentType("application/json");
+ resp.getWriter().print(createStringFromErrorMessageObject(errorResponse));
+ return;
+ }
+ }
+ }
+
+ filterChain.doFilter(servletRequest, servletResponse);
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ private VResponse getErrorResponse() {
+ if (!solrLogLevelFilterManagerState.isLogLevelFilterManagerIsReady()) {
+ return RESTErrorUtil.createMessageResponse(SOLR_FILTER_MANAGER_NOT_AVAILABLE, MessageEnums.SOLR_CONFIGURATION_API_SOLR_NOT_AVAILEBLE);
+ }
+
+ return null;
+ }
+
+ private String createStringFromErrorMessageObject(VResponse responseObject) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(responseObject);
+ } catch (Exception e) {
+ throw RESTErrorUtil.createRESTException("Cannot parse response object on backend", MessageEnums.ERROR_CREATING_OBJECT);
+ }
+ }
+}
diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
index 850aca2..20aed68 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
+++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
@@ -30,4 +30,6 @@ logfeeder.cache.dedup.interval=1000
logfeeder.cache.last.dedup.enabled=true
logsearch.config.zk_connect_string=localhost:9983
logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
-logfeeder.docker.registry.enabled=true
\ No newline at end of file
+logfeeder.docker.registry.enabled=true
+#logfeeder.configs.local.enabled=true
+#logfeeder.configs.filter.solr.enabled=true
\ No newline at end of file
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json
index f41e981..a85b4a4 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json
@@ -3,6 +3,7 @@
{
"is_enabled": "true",
"comment": "Output to solr for service logs",
+ "collection" : "hadoop_logs",
"destination": "solr",
"zk_connect_string": "localhost:9983",
"type": "service",
@@ -18,6 +19,7 @@
{
"comment": "Output to solr for audit records",
"is_enabled": "true",
+ "collection" : "audit_logs",
"destination": "solr",
"zk_connect_string": "localhost:9983",
"type": "audit",
diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
index 22edf76..21bc797 100644
--- a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
+++ b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
@@ -18,6 +18,9 @@ logsearch.solr.zk_connect_string=localhost:9983
# Service Logs
logsearch.solr.collection.service.logs=hadoop_logs
+#logsearch.config.api.filter.solr.enabled=true
+#logsearch.config.api.enabled=false
+
logsearch.service.logs.split.interval.mins=15
logsearch.collection.service.logs.numshards=3
logsearch.collection.service.logs.replication.factor=2
diff --git a/ambari-logsearch/pom.xml b/ambari-logsearch/pom.xml
index 6d5f9f1..cbe644b 100644
--- a/ambari-logsearch/pom.xml
+++ b/ambari-logsearch/pom.xml
@@ -30,10 +30,13 @@
<module>ambari-logsearch-web</module>
<module>ambari-logsearch-logfeeder</module>
<module>ambari-logsearch-config-api</module>
+ <module>ambari-logsearch-config-json</module>
<module>ambari-logsearch-config-zookeeper</module>
<module>ambari-logsearch-it</module>
<module>ambari-logsearch-logfeeder-plugin-api</module>
<module>ambari-logsearch-logfeeder-container-registry</module>
+ <module>ambari-logsearch-config-local</module>
+ <module>ambari-logsearch-config-solr</module>
</modules>
<properties>
<jdk.version>1.8</jdk.version>
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml
index 652918a..29999ba 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml
@@ -32,6 +32,6 @@
<property-file-name>output.config.json.j2</property-file-name>
<property-file-type>text</property-file-type>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="true" update="true"/>
</property>
</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
index 0c599c9..ffcfeaf 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
@@ -21,6 +21,7 @@
"comment":"Output to solr for service logs",
"is_enabled":"{{solr_service_logs_enable}}",
"destination":"solr",
+ "collection": "{{logsearch_solr_collection_service_logs}}",
"zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}",
"type": "service",
"conditions":{
@@ -38,6 +39,7 @@
"comment":"Output to solr for audit records",
"is_enabled":"{{solr_audit_logs_enable}}",
"destination":"solr",
+ "collection": "{{logsearch_solr_collection_audit_logs}}",
"zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}",
"type": "audit",
"conditions":{