You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/08/08 16:02:40 UTC
[24/50] [abbrv] ambari git commit: AMBARI-21623 Log Search Config
should be separated into a Server and Log Feeder interface (mgergely)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
index 5fdfded..8f8c4fd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
@@ -27,8 +27,8 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
import org.apache.log4j.Logger;
import com.google.common.io.Files;
@@ -47,13 +47,13 @@ public class InputConfigUploader extends Thread {
};
private final Set<String> filesHandled = new HashSet<>();
private final Pattern serviceNamePattern = Pattern.compile("input.config-(.+).json");
- private final LogSearchConfig config;
+ private final LogSearchConfigLogFeeder config;
- public static void load(LogSearchConfig config) {
+ public static void load(LogSearchConfigLogFeeder config) {
new InputConfigUploader(config).start();
}
- private InputConfigUploader(LogSearchConfig config) {
+ private InputConfigUploader(LogSearchConfigLogFeeder config) {
super("Input Config Loader");
setDaemon(true);
@@ -73,7 +73,7 @@ public class InputConfigUploader extends Thread {
String serviceName = m.group(1);
String inputConfig = Files.toString(inputConfigFile, Charset.defaultCharset());
- if (!config.inputConfigExistsLogFeeder(serviceName)) {
+ if (!config.inputConfigExists(serviceName)) {
config.createInputConfig(LogFeederPropertiesUtil.getClusterName(), serviceName, inputConfig);
}
filesHandled.add(inputConfigFile.getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
index b370e58..c0075b5 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
@@ -28,7 +28,7 @@ import org.apache.ambari.logfeeder.common.ConfigBlock;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
import org.apache.ambari.logsearch.config.api.OutputConfigMonitor;
import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
@@ -104,9 +104,9 @@ public abstract class Output extends ConfigBlock implements OutputConfigMonitor
this.destination = destination;
}
- protected LogSearchConfig logSearchConfig;
+ protected LogSearchConfigLogFeeder logSearchConfig;
- public void setLogSearchConfig(LogSearchConfig logSearchConfig) {
+ public void setLogSearchConfig(LogSearchConfigLogFeeder logSearchConfig) {
this.logSearchConfig = logSearchConfig;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
index ce040f9..5ab271a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
@@ -28,7 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl.OutputSolrPropertiesImpl;
import org.apache.log4j.Logger;
@@ -50,7 +50,7 @@ public class OutputSolrTest {
private static final Logger LOG = Logger.getLogger(OutputSolrTest.class);
private OutputSolr outputSolr;
- private LogSearchConfig logSearchConfigMock;
+ private LogSearchConfigLogFeeder logSearchConfigMock;
private Map<Integer, SolrInputDocument> receivedDocs = new ConcurrentHashMap<>();
@Rule
@@ -80,7 +80,7 @@ public class OutputSolrTest {
};
OutputSolrProperties outputSolrProperties = new OutputSolrPropertiesImpl("hadoop_logs", "none");
- logSearchConfigMock = EasyMock.createNiceMock(LogSearchConfig.class);
+ logSearchConfigMock = EasyMock.createNiceMock(LogSearchConfigLogFeeder.class);
EasyMock.expect(logSearchConfigMock.getOutputSolrProperties("service")).andReturn(outputSolrProperties);
EasyMock.replay(logSearchConfigMock);
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
index 3f6df75..23b7430 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
@@ -25,10 +25,9 @@ import javax.inject.Named;
import org.apache.ambari.logsearch.common.PropertiesHelper;
import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
import org.apache.ambari.logsearch.config.api.LogSearchConfigFactory;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig.Component;
-import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigZK;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
+import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigServerZK;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,8 +37,8 @@ public class LogSearchConfigConfigurer implements Configurer {
private static final int RETRY_INTERVAL_SECONDS = 10;
- private static LogSearchConfig logSearchConfig;
- public static LogSearchConfig getConfig() {
+ private LogSearchConfigServer logSearchConfig;
+ public LogSearchConfigServer getConfig() {
return logSearchConfig;
}
@@ -55,8 +54,8 @@ public class LogSearchConfigConfigurer implements Configurer {
logger.info("Started thread to set up log search config");
while (true) {
try {
- logSearchConfig = LogSearchConfigFactory.createLogSearchConfig(Component.SERVER, PropertiesHelper.getProperties(),
- null, LogSearchConfigZK.class);
+ logSearchConfig = LogSearchConfigFactory.createLogSearchConfigServer(PropertiesHelper.getProperties(),
+ LogSearchConfigServerZK.class);
logSearchConfigState.setLogSearchConfigAvailable(true);
break;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
index 6de34ce..ff828a3 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
@@ -219,7 +219,8 @@ public class SolrCollectionConfigurer implements Configurer {
boolean reloadCollectionNeeded) {
try {
List<String> allCollectionList = new ListCollectionHandler().handle(solrClient, null);
- boolean collectionCreated = new CreateCollectionHandler(allCollectionList).handle(solrClient, solrPropsConfig);
+ CreateCollectionHandler handler = new CreateCollectionHandler(solrDaoBase.getLogSearchConfig(), allCollectionList);
+ boolean collectionCreated = handler.handle(solrClient, solrPropsConfig);
boolean collectionReloaded = true;
if (reloadCollectionNeeded) {
collectionReloaded = new ReloadCollectionHandler().handle(solrClient, solrPropsConfig);
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 15f59e4..ef5e697 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -26,6 +26,8 @@ import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
+import org.apache.ambari.logsearch.configurer.LogSearchConfigConfigurer;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.log4j.Logger;
@@ -56,7 +58,10 @@ public abstract class SolrDaoBase {
private SolrKerberosConfig solrKerberosConfig;
@Inject
- protected LogSearchConfigState logSearchConfigState;
+ private LogSearchConfigState logSearchConfigState;
+
+ @Inject
+ private LogSearchConfigConfigurer logSearchConfigConfigurer;
protected SolrDaoBase(LogType logType) {
this.logType = logType;
@@ -152,6 +157,10 @@ public abstract class SolrDaoBase {
return (CloudSolrClient) getSolrTemplate().getSolrClient();
}
+ public LogSearchConfigServer getLogSearchConfig() {
+ return logSearchConfigConfigurer.getConfig();
+ }
+
public abstract SolrTemplate getSolrTemplate();
public abstract void setSolrTemplate(SolrTemplate solrTemplate);
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index 8b4c7bf..a988856 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logsearch.handler;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigServer;
import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
import org.apache.ambari.logsearch.config.zookeeper.model.outputconfig.impl.OutputSolrPropertiesImpl;
import org.apache.ambari.logsearch.configurer.LogSearchConfigConfigurer;
@@ -54,9 +55,11 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
+ private final LogSearchConfigServer logSearchConfig;
private final List<String> allCollectionList;
- public CreateCollectionHandler(List<String> allCollectionList) {
+ public CreateCollectionHandler(LogSearchConfigServer logSearchConfig, List<String> allCollectionList) {
+ this.logSearchConfig = logSearchConfig;
this.allCollectionList = allCollectionList;
}
@@ -65,7 +68,7 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
if (solrPropsConfig.getLogType() != null) {
OutputSolrProperties outputSolrProperties = new OutputSolrPropertiesImpl(solrPropsConfig.getCollection(),
solrPropsConfig.getSplitInterval());
- LogSearchConfigConfigurer.getConfig().saveOutputSolrProperties(solrPropsConfig.getLogType(), outputSolrProperties);
+ logSearchConfig.saveOutputSolrProperties(solrPropsConfig.getLogType(), outputSolrProperties);
}
boolean result;
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
index a1181b4..b19db72 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
@@ -50,24 +50,24 @@ public class ShipperConfigManager extends JsonManagerBase {
private LogSearchConfigConfigurer logSearchConfigConfigurer;
public List<String> getServices(String clusterName) {
- return LogSearchConfigConfigurer.getConfig().getServices(clusterName);
+ return logSearchConfigConfigurer.getConfig().getServices(clusterName);
}
public LSServerInputConfig getInputConfig(String clusterName, String serviceName) {
- InputConfig inputConfig = LogSearchConfigConfigurer.getConfig().getInputConfig(clusterName, serviceName);
+ InputConfig inputConfig = logSearchConfigConfigurer.getConfig().getInputConfig(clusterName, serviceName);
return new LSServerInputConfig(inputConfig);
}
public Response createInputConfig(String clusterName, String serviceName, LSServerInputConfig inputConfig) {
try {
- if (LogSearchConfigConfigurer.getConfig().inputConfigExistsServer(clusterName, serviceName)) {
+ if (logSearchConfigConfigurer.getConfig().inputConfigExists(clusterName, serviceName)) {
return Response.serverError()
.type(MediaType.APPLICATION_JSON)
.entity(ImmutableMap.of("errorMessage", "Input config already exists for service " + serviceName))
.build();
}
- LogSearchConfigConfigurer.getConfig().createInputConfig(clusterName, serviceName, new ObjectMapper().writeValueAsString(inputConfig));
+ logSearchConfigConfigurer.getConfig().createInputConfig(clusterName, serviceName, new ObjectMapper().writeValueAsString(inputConfig));
return Response.ok().build();
} catch (Exception e) {
logger.warn("Could not create input config", e);
@@ -77,14 +77,14 @@ public class ShipperConfigManager extends JsonManagerBase {
public Response setInputConfig(String clusterName, String serviceName, LSServerInputConfig inputConfig) {
try {
- if (!LogSearchConfigConfigurer.getConfig().inputConfigExistsServer(clusterName, serviceName)) {
+ if (!logSearchConfigConfigurer.getConfig().inputConfigExists(clusterName, serviceName)) {
return Response.serverError()
.type(MediaType.APPLICATION_JSON)
.entity(ImmutableMap.of("errorMessage", "Input config doesn't exist for service " + serviceName))
.build();
}
- LogSearchConfigConfigurer.getConfig().setInputConfig(clusterName, serviceName, new ObjectMapper().writeValueAsString(inputConfig));
+ logSearchConfigConfigurer.getConfig().setInputConfig(clusterName, serviceName, new ObjectMapper().writeValueAsString(inputConfig));
return Response.ok().build();
} catch (Exception e) {
logger.warn("Could not update input config", e);
@@ -101,7 +101,7 @@ public class ShipperConfigManager extends JsonManagerBase {
throw new IllegalArgumentException("Error validating shipper config:\n" + violations);
}
- String globalConfigs = LogSearchConfigConfigurer.getConfig().getGlobalConfigs(clusterName);
+ String globalConfigs = logSearchConfigConfigurer.getConfig().getGlobalConfigs(clusterName);
LogEntryParseTester tester = new LogEntryParseTester(testEntry, shipperConfig, globalConfigs, logId);
return tester.parse();
} catch (Exception e) {
@@ -111,12 +111,12 @@ public class ShipperConfigManager extends JsonManagerBase {
}
public LSServerLogLevelFilterMap getLogLevelFilters(String clusterName) {
- return new LSServerLogLevelFilterMap(LogSearchConfigConfigurer.getConfig().getLogLevelFilters(clusterName));
+ return new LSServerLogLevelFilterMap(logSearchConfigConfigurer.getConfig().getLogLevelFilters(clusterName));
}
public Response setLogLevelFilters(String clusterName, LSServerLogLevelFilterMap request) {
try {
- LogSearchConfigConfigurer.getConfig().setLogLevelFilters(clusterName, request.convertToApi());
+ logSearchConfigConfigurer.getConfig().setLogLevelFilters(clusterName, request.convertToApi());
return Response.ok().build();
} catch (Exception e) {
logger.warn("Could not update log level filters", e);
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d1ac668/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
index 84bddb8..72cc72b 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java
@@ -70,10 +70,12 @@ public abstract class LSServerFilter {
this.sourceField = filterDescriptor.getSourceField();
this.removeSourceField = filterDescriptor.isRemoveSourceField();
- this.postMapValues = new HashMap<String, LSServerPostMapValuesList>();
- for (Map.Entry<String, ? extends List<? extends PostMapValues>> e : filterDescriptor.getPostMapValues().entrySet()) {
- LSServerPostMapValuesList lsServerPostMapValuesList = new LSServerPostMapValuesList(e.getValue());
- postMapValues.put(e.getKey(), lsServerPostMapValuesList);
+ if (filterDescriptor.getPostMapValues() != null) {
+ this.postMapValues = new HashMap<String, LSServerPostMapValuesList>();
+ for (Map.Entry<String, ? extends List<? extends PostMapValues>> e : filterDescriptor.getPostMapValues().entrySet()) {
+ LSServerPostMapValuesList lsServerPostMapValuesList = new LSServerPostMapValuesList(e.getValue());
+ postMapValues.put(e.getKey(), lsServerPostMapValuesList);
+ }
}
this.isEnabled = filterDescriptor.isEnabled();