You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2017/01/24 22:15:44 UTC
ambari git commit: AMBARI-19658. LogSearch Integration Cache Timeout
should be configurable. (rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk 78fefdd4c -> 604620021
AMBARI-19658. LogSearch Integration Cache Timeout should be configurable. (rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60462002
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60462002
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60462002
Branch: refs/heads/trunk
Commit: 604620021908995d7b4581176ceecf6c44ffea26
Parents: 78fefdd
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Tue Jan 24 17:15:07 2017 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Tue Jan 24 17:15:29 2017 -0500
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 26 +++++++++++++++++-
.../logging/LogSearchDataRetrievalService.java | 28 +++++++++++++++++---
.../LogSearchDataRetrievalServiceTest.java | 27 ++++++++++++++++++-
3 files changed, 76 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index df1b627..73c70dc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2601,7 +2601,6 @@ public class Configuration {
public static final ConfigurationProperty<Integer> LOGSEARCH_PORTAL_READ_TIMEOUT = new ConfigurationProperty<>(
"logsearch.portal.read.timeout", 5000);
-
/**
* Global disable flag for AmbariServer Metrics.
*/
@@ -2609,6 +2608,18 @@ public class Configuration {
public static final ConfigurationProperty<Boolean> AMBARISERVER_METRICS_DISABLE = new ConfigurationProperty<>(
"ambariserver.metrics.disable", false);
+ /**
+ * The time, in hours, that the Ambari Server will hold Log File metadata in its internal cache before making
+ * a request to the LogSearch Portal to get the latest metadata.
+ *
+ * The logging metadata (in this case, log file names) is generally quite static, so the default should
+ * generally be quite long.
+ *
+ */
+ @Markdown(description = "The time, in hours, that the Ambari Server will hold Log File metadata in its internal cache before making a request to the LogSearch Portal to get the latest metadata.")
+ public static final ConfigurationProperty<Integer> LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT = new ConfigurationProperty<>(
+ "logsearch.metadata.cache.expire.timeout", 24);
+
private static final Logger LOG = LoggerFactory.getLogger(
Configuration.class);
@@ -5406,6 +5417,19 @@ public class Configuration {
return NumberUtils.toInt(getProperty(LOGSEARCH_PORTAL_READ_TIMEOUT));
}
+
+ /**
+ *
+ * Get the max time, in hours, to hold data in the LogSearch
+ * metadata cache prior to expiring the cache and re-loading
+ * the data from the LogSearch Portal service.
+ *
+ * @return max number of hours that the LogSearch metadata is cached
+ */
+ public int getLogSearchMetadataCacheExpireTimeout() {
+ return NumberUtils.toInt(getProperty(LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT));
+ }
+
/**
* Generates a markdown table which includes:
* <ul>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
index ce6094c..6b484a4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
@@ -23,6 +23,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.ambari.server.AmbariService;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.commons.collections.CollectionUtils;
@@ -77,6 +78,11 @@ public class LogSearchDataRetrievalService extends AbstractService {
@Inject
private Injector injector;
+ @Inject
+ private Configuration ambariServerConfiguration;
+
+
+
/**
* A Cache of host+component names to a set of log files associated with
* that Host/Component combination. This data is retrieved from the
@@ -114,10 +120,17 @@ public class LogSearchDataRetrievalService extends AbstractService {
protected void doStart() {
LOG.debug("Initializing caches");
+
+ // obtain the max cache expire time from the ambari configuration
+ final int maxTimeoutForCacheInHours =
+ ambariServerConfiguration.getLogSearchMetadataCacheExpireTimeout();
+
+ LOG.debug("Caches configured with a max expire timeout of " + maxTimeoutForCacheInHours + " hours.");
+
// initialize the log file name cache
- logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build();
+ logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours, TimeUnit.HOURS).build();
// initialize the log file tail URI cache
- logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build();
+ logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours, TimeUnit.HOURS).build();
// initialize the Executor
executor = Executors.newSingleThreadExecutor();
@@ -126,7 +139,7 @@ public class LogSearchDataRetrievalService extends AbstractService {
@Override
protected void doStop() {
LOG.debug("Invalidating LogSearch caches");
- // invalidate the cache
+ // invalidate the caches
logFileNameCache.invalidateAll();
logFileTailURICache.invalidateAll();
@@ -230,6 +243,15 @@ public class LogSearchDataRetrievalService extends AbstractService {
}
/**
+ * Package-level setter to facilitate simpler unit testing
+ *
+ * @param ambariServerConfiguration
+ */
+ void setConfiguration(Configuration ambariServerConfiguration) {
+ this.ambariServerConfiguration = ambariServerConfiguration;
+ }
+
+ /**
* This protected method allows for simpler unit tests.
*
* @return the Set of current Requests that are not yet completed
http://git-wip-us.apache.org/repos/asf/ambari/blob/60462002/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
index 033d698..4296004 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java
@@ -29,6 +29,7 @@ import java.util.Set;
import java.util.concurrent.Executor;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.easymock.EasyMockSupport;
import org.junit.Test;
@@ -60,13 +61,18 @@ public class LogSearchDataRetrievalServiceTest {
LoggingRequestHelper helperMock =
mockSupport.createMock(LoggingRequestHelper.class);
+ Configuration configurationMock =
+ mockSupport.createMock(Configuration.class);
+
expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(helperMock);
expect(helperMock.createLogFileTailURI("http://localhost", expectedComponentName, expectedHostName)).andReturn(expectedResultURI);
+ expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
mockSupport.replayAll();
LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
+ retrievalService.setConfiguration(configurationMock);
// call the initialization routine called by the Google framework
retrievalService.doStart();
@@ -86,16 +92,23 @@ public class LogSearchDataRetrievalServiceTest {
EasyMockSupport mockSupport = new EasyMockSupport();
- LoggingRequestHelperFactory helperFactoryMock = mockSupport.createMock(LoggingRequestHelperFactory.class);
+ LoggingRequestHelperFactory helperFactoryMock =
+ mockSupport.createMock(LoggingRequestHelperFactory.class);
+
+ Configuration configurationMock =
+ mockSupport.createMock(Configuration.class);
// return null, to simulate the case where LogSearch Server is
// not available for some reason
expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(null);
+ expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
+
mockSupport.replayAll();
LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
+ retrievalService.setConfiguration(configurationMock);
// call the initialization routine called by the Google framework
retrievalService.doStart();
@@ -122,6 +135,9 @@ public class LogSearchDataRetrievalServiceTest {
Injector injectorMock =
mockSupport.createMock(Injector.class);
+ Configuration configurationMock =
+ mockSupport.createMock(Configuration.class);
+
// expect the executor to be called to execute the LogSearch request
executorMock.execute(isA(LogSearchDataRetrievalService.LogSearchFileNameRequestRunnable.class));
// executor should only be called once
@@ -129,11 +145,14 @@ public class LogSearchDataRetrievalServiceTest {
expect(injectorMock.getInstance(LoggingRequestHelperFactory.class)).andReturn(helperFactoryMock);
+ expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
+
mockSupport.replayAll();
LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
retrievalService.setInjector(injectorMock);
+ retrievalService.setConfiguration(configurationMock);
// call the initialization routine called by the Google framework
retrievalService.doStart();
retrievalService.setExecutor(executorMock);
@@ -164,10 +183,16 @@ public class LogSearchDataRetrievalServiceTest {
Executor executorMock = mockSupport.createMock(Executor.class);
+ Configuration configurationMock =
+ mockSupport.createMock(Configuration.class);
+
+ expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce();
+
mockSupport.replayAll();
LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService();
retrievalService.setLoggingRequestHelperFactory(helperFactoryMock);
+ retrievalService.setConfiguration(configurationMock);
// call the initialization routine called by the Google framework
retrievalService.doStart();
// there should be no expectations set on this mock