You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/04/04 10:57:50 UTC
[ambari] branch trunk updated: AMBARI-23080. Log Search: use
compositeId instead of implicit routing by default for collections. (#866)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new e2832d0 AMBARI-23080. Log Search: use compositeId instead of implicit routing by default for collections. (#866)
e2832d0 is described below
commit e2832d0879a5007a4dac6e99cd15f5e727430769
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Wed Apr 4 12:57:47 2018 +0200
AMBARI-23080. Log Search: use compositeId instead of implicit routing by default for collections. (#866)
* AMBARI-23080. Log Search: use compositeId instead of implicit routing by default for collections.
* AMBARI-23080. Use implicitRouting local variable.
---
.../ambari/logfeeder/common/LogFeederConstants.java | 1 +
.../apache/ambari/logfeeder/conf/LogFeederProps.java | 18 ++++++++++++++++++
.../apache/ambari/logfeeder/output/OutputSolr.java | 16 +++++++++++-----
.../logsearch/conf/SolrAuditLogPropsConfig.java | 20 ++++++++++++++++++++
.../logsearch/conf/SolrConnectionPropsConfig.java | 20 ++++++++++++++++++++
.../ambari/logsearch/conf/SolrPropsConfig.java | 4 ++++
.../logsearch/handler/CreateCollectionHandler.java | 6 +++---
.../0.5.0/configuration/logfeeder-properties.xml | 12 ++++++++++++
.../0.5.0/configuration/logsearch-properties.xml | 12 ++++++++++++
9 files changed, 101 insertions(+), 8 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index b241831..80dc163 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -50,6 +50,7 @@ public class LogFeederConstants {
public static final String LOG_FILTER_ENABLE_PROPERTY = "logfeeder.log.filter.enable";
public static final String INCLUDE_DEFAULT_LEVEL_PROPERTY = "logfeeder.include.default.level";
+ public static final String SOLR_IMPLICIT_ROUTING_PROPERTY = "logfeeder.solr.implicit.routing";
public static final String CONFIG_DIR_PROPERTY = "logfeeder.config.dir";
public static final String CONFIG_FILES_PROPERTY = "logfeeder.config.files";
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index cc1f64c..9a40e70 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -74,6 +74,16 @@ public class LogFeederProps implements LogFeederProperties {
private boolean logLevelFilterEnabled;
@LogSearchPropertyDescription(
+ name = LogFeederConstants.SOLR_IMPLICIT_ROUTING_PROPERTY,
+ description = "Use implicit routing for Solr Collections.",
+ examples = {"true"},
+ defaultValue = "false",
+ sources = {LogFeederConstants.SOLR_IMPLICIT_ROUTING_PROPERTY}
+ )
+ @Value("${"+ LogFeederConstants.SOLR_IMPLICIT_ROUTING_PROPERTY + ":false}")
+ private boolean solrImplicitRouting;
+
+ @LogSearchPropertyDescription(
name = LogFeederConstants.INCLUDE_DEFAULT_LEVEL_PROPERTY,
description = "Comma separated list of the default log levels to be enabled by the filtering.",
examples = {"FATAL,ERROR,WARN"},
@@ -209,6 +219,14 @@ public class LogFeederProps implements LogFeederProperties {
this.checkpointFolder = checkpointFolder;
}
+ public boolean isSolrImplicitRouting() {
+ return solrImplicitRouting;
+ }
+
+ public void setSolrImplicitRouting(boolean solrImplicitRouting) {
+ this.solrImplicitRouting = solrImplicitRouting;
+ }
+
@PostConstruct
public void init() {
properties = new Properties();
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index 160085c..12a804d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -81,7 +81,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
private int maxIntervalMS;
private int workers;
private int maxBufferSize;
- private boolean isComputeCurrentCollection = false;
+ private boolean implicitRouting = false;
private int lastSlotByMin = -1;
private boolean skipLogtime = false;
@@ -115,14 +115,14 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
@Override
public void init(LogFeederProps logFeederProps) throws Exception {
this.logFeederProps = logFeederProps;
- initParams();
+ initParams(logFeederProps);
setupSecurity();
createOutgoingBuffer();
createSolrStateWatcher();
createSolrWorkers();
}
- private void initParams() throws Exception {
+ private void initParams(LogFeederProps logFeederProps) throws Exception {
type = getStringValue("type");
while (true) {
OutputSolrProperties outputSolrProperties = getLogSearchConfig().getOutputSolrProperties(type);
@@ -153,6 +153,13 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
LOG.info(String.format("Config: Number of workers=%d, splitMode=%s, splitInterval=%d."
+ getShortDescription(), workers, splitMode, splitInterval));
+
+ implicitRouting = logFeederProps.isSolrImplicitRouting(); // TODO: in the future, load it from output config (can be a use case to use different routing for audit/service logs)
+ if (implicitRouting) {
+ LOG.info("Config: Use implicit routing globally for adding docs to Solr.");
+ } else {
+ LOG.info("Config: Use compositeId globally for adding docs to Solr.");
+ }
}
@Override
@@ -166,7 +173,6 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
if (!splitMode.equalsIgnoreCase("none")) {
splitInterval = Integer.parseInt(splitMode);
}
- isComputeCurrentCollection = !splitMode.equalsIgnoreCase("none");
// collection can not be overwritten after initialization
if (init) {
@@ -424,7 +430,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> implements C
while (!isDrain()) {
try {
synchronized (propertiesLock) {
- if (isComputeCurrentCollection) {
+ if (implicitRouting) {
// Compute the current router value
addRouterField();
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
index e477fc6..43349a6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
@@ -136,6 +136,16 @@ public class SolrAuditLogPropsConfig implements SolrPropsConfig {
)
private String configSetFolder;
+ @LogSearchPropertyDescription(
+ name = "logsearch.solr.implicit.routing",
+ description = "Use implicit routing for Solr Collections.",
+ examples = {"true"},
+ defaultValue = "false",
+ sources = {LOGSEARCH_PROPERTIES_FILE}
+ )
+ @Value("${logsearch.solr.implicit.routing:false}")
+ private boolean solrImplicitRouting;
+
@Override
public String getSolrUrl() {
return solrUrl;
@@ -243,6 +253,16 @@ public class SolrAuditLogPropsConfig implements SolrPropsConfig {
}
@Override
+ public boolean isSolrImplicitRouting() {
+ return solrImplicitRouting;
+ }
+
+ @Override
+ public void setSolrImplicitRouting(boolean solrImplicitRouting) {
+ this.solrImplicitRouting = solrImplicitRouting;
+ }
+
+ @Override
public String getLogType() {
return "audit";
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
index 818cc04..87b77bf 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
@@ -64,6 +64,16 @@ public abstract class SolrConnectionPropsConfig implements SolrPropsConfig {
)
private String configSetFolder;
+ @LogSearchPropertyDescription(
+ name = "logsearch.solr.implicit.routing",
+ description = "Use implicit routing for Solr Collections.",
+ examples = {"true"},
+ defaultValue = "false",
+ sources = {LOGSEARCH_PROPERTIES_FILE}
+ )
+ @Value("${logsearch.solr.implicit.routing:false}")
+ private boolean solrImplicitRouting;
+
@Override
public String getSolrUrl() {
return solrUrl;
@@ -103,4 +113,14 @@ public abstract class SolrConnectionPropsConfig implements SolrPropsConfig {
public void setConfigSetFolder(String configSetFolder) {
this.configSetFolder = configSetFolder;
}
+
+ @Override
+ public boolean isSolrImplicitRouting() {
+ return solrImplicitRouting;
+ }
+
+ @Override
+ public void setSolrImplicitRouting(boolean solrImplicitRouting) {
+ this.solrImplicitRouting = solrImplicitRouting;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
index cd0a1c2..ebb1acb 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
@@ -60,4 +60,8 @@ public interface SolrPropsConfig {
void setConfigSetFolder(String configSetFolder);
String getLogType();
+
+ boolean isSolrImplicitRouting();
+
+ void setSolrImplicitRouting(boolean solrImplicitRouting);
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index c3a5956..ba5432f 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -71,10 +71,10 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
}
boolean result;
- if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
- result = createCollection(solrClient, solrPropsConfig, this.allCollectionList);
- } else {
+ if (solrPropsConfig.isSolrImplicitRouting()) {
result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, this.allCollectionList);
+ } else {
+ result = createCollection(solrClient, solrPropsConfig, this.allCollectionList);
}
return result;
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
index b43aaa6..074717a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
@@ -65,6 +65,18 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>logfeeder.solr.implicit.routing</name>
+ <value>false</value>
+ <description>
+ Use implicit routing for Solr Collections.
+ </description>
+ <display-name>Log Feeder Solr implicit routing</display-name>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>logfeeder.cache.enabled</name>
<value>false</value>
<description>
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
index 0bac71b..24947bb 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
@@ -134,6 +134,18 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>logsearch.solr.implicit.routing</name>
+ <value>false</value>
+ <description>
+ Use implicit routing for Solr Collections.
+ </description>
+ <display-name>Log Search Solr implicit routing</display-name>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>logsearch.solr.audit.logs.use.ranger</name>
<value>false</value>
<display-name>Ranger Audit Logs Enabled</display-name>
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.