You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by am...@apache.org on 2017/12/22 09:09:46 UTC
[01/51] [abbrv] ambari git commit: AMBARI-22639. Log Feeder refactor:
integrate with spring boot (oleewere) [Forced Update!]
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-22008-isilon 37a2ca707 -> b32efecaf (forced update)
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java
deleted file mode 100644
index 46abc63..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.logfeeder.logconfig;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.*;
-
-import org.apache.ambari.logfeeder.input.Input;
-import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.loglevelfilter.FilterLogData;
-import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
-import org.apache.ambari.logsearch.config.api.LogSearchConfig;
-import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
-import org.apache.commons.lang.time.DateUtils;
-import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class LogConfigHandlerTest {
-
- private static InputMarker inputMarkerAudit;
- private static InputMarker inputMarkerService;
- static {
- InputDescriptorImpl auditInputDescriptor = new InputDescriptorImpl() {};
- auditInputDescriptor.setRowtype("audit");
-
- Input auditInput = strictMock(Input.class);
- expect(auditInput.getInputDescriptor()).andReturn(auditInputDescriptor).anyTimes();
- inputMarkerAudit = new InputMarker(auditInput, null, 0);
-
- InputDescriptorImpl serviceInputDescriptor = new InputDescriptorImpl() {};
- serviceInputDescriptor.setRowtype("service");
-
- Input serviceInput = strictMock(Input.class);
- expect(serviceInput.getInputDescriptor()).andReturn(serviceInputDescriptor).anyTimes();
- inputMarkerService = new InputMarker(serviceInput, null, 0);
-
- replay(auditInput, serviceInput);
- }
-
- @BeforeClass
- public static void init() throws Exception {
- LogFeederPropertiesUtil.loadProperties("logfeeder.properties");
-
- LogSearchConfig config = strictMock(LogSearchConfig.class);
- config.createLogLevelFilter(anyString(), anyString(), anyObject(LogLevelFilter.class));
- expectLastCall().anyTimes();
- LogLevelFilterHandler.init(config);
-
- LogLevelFilter logLevelFilter1 = new LogLevelFilter();
- logLevelFilter1.setHosts(Collections.<String> emptyList());
- logLevelFilter1.setDefaultLevels(Arrays.asList("FATAL", "ERROR", "WARN", "INFO"));
- logLevelFilter1.setOverrideLevels(Collections.<String> emptyList());
-
- LogLevelFilter logLevelFilter2 = new LogLevelFilter();
- logLevelFilter2.setHosts(Arrays.asList("host1"));
- logLevelFilter2.setDefaultLevels(Arrays.asList("FATAL", "ERROR", "WARN", "INFO"));
- logLevelFilter2.setOverrideLevels(Arrays.asList("FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"));
- logLevelFilter2.setExpiryTime(DateUtils.addDays(new Date(), 1));
-
- LogLevelFilter logLevelFilter3 = new LogLevelFilter();
- logLevelFilter3.setHosts(Arrays.asList("host1"));
- logLevelFilter3.setDefaultLevels(Arrays.asList("FATAL", "ERROR", "WARN", "INFO"));
- logLevelFilter3.setOverrideLevels(Arrays.asList("FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"));
- logLevelFilter3.setExpiryTime(DateUtils.addDays(new Date(), -1));
-
- LogLevelFilterHandler h = new LogLevelFilterHandler();
- h.setLogLevelFilter("configured_log_file1", logLevelFilter1);
- h.setLogLevelFilter("configured_log_file2", logLevelFilter2);
- h.setLogLevelFilter("configured_log_file3", logLevelFilter3);
- }
-
- @Test
- public void testLogConfigHandler_auditAllowed() throws Exception {
- assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file1', 'level':'DEBUG'}",
- inputMarkerAudit));
- }
-
- @Test
- public void testLogConfigHandler_emptyDataAllowed() throws Exception {
- assertTrue(FilterLogData.INSTANCE.isAllowed((String)null, inputMarkerService));
- assertTrue(FilterLogData.INSTANCE.isAllowed("", inputMarkerService));
- assertTrue(FilterLogData.INSTANCE.isAllowed(Collections.<String, Object> emptyMap(), inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_notConfiguredLogAllowed() throws Exception {
- assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'not_configured_log_file1', 'level':'WARN'}",
- inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_notConfiguredLogNotAllowed() throws Exception {
- assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'not_configured_log_file1', 'level':'TRACE'}",
- inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_configuredDataAllow() throws Exception {
- assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file1', 'level':'INFO'}",
- inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_configuredDataDontAllow() throws Exception {
- assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file1', 'level':'DEBUG'}",
- inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_overridenConfiguredData() throws Exception {
- assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file2', 'level':'DEBUG'}",
- inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_overridenConfiguredDataDifferentHost() throws Exception {
- assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host2', 'type':'configured_log_file2', 'level':'DEBUG'}",
- inputMarkerService));
- }
-
- @Test
- public void testLogConfigHandler_overridenConfiguredDataExpired() throws Exception {
- assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file3', 'level':'DEBUG'}",
- inputMarkerService));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
index f74a80e..da8fff7 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
@@ -29,11 +29,9 @@ import java.util.Arrays;
import java.util.List;
import java.util.TreeMap;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
public class MetricsManagerTest {
@@ -42,11 +40,6 @@ public class MetricsManagerTest {
private LogFeederAMSClient mockClient;
private Capture<TimelineMetrics> capture;
- @BeforeClass
- public static void loadProperties() throws Exception {
- LogFeederPropertiesUtil.loadProperties("logfeeder.properties");
- }
-
@Before
public void init() throws Exception {
manager = new MetricsManager();
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
index 38d4b8b..c0babc4 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Future;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.OutputKafka.KafkaCallBack;
@@ -69,7 +70,7 @@ public class OutputKafkaTest {
config.put("topic", TEST_TOPIC);
outputKafka.loadConfig(config);
- outputKafka.init();
+ outputKafka.init(new LogFeederProps());
@SuppressWarnings("unchecked")
Future<RecordMetadata> mockFuture = EasyMock.mock(Future.class);
@@ -103,7 +104,7 @@ public class OutputKafkaTest {
config.put("topic", TEST_TOPIC);
outputKafka.loadConfig(config);
- outputKafka.init();
+ outputKafka.init(new LogFeederProps());
}
@Test
@@ -117,7 +118,7 @@ public class OutputKafkaTest {
config.put("broker_list", "some broker list");
outputKafka.loadConfig(config);
- outputKafka.init();
+ outputKafka.init(new LogFeederProps());
}
@After
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
index 5abb720..49f5a11 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
@@ -29,8 +29,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl;
import org.junit.Test;
@@ -63,10 +65,11 @@ public class OutputManagerTest {
Output output1 = strictMock(Output.class);
Output output2 = strictMock(Output.class);
Output output3 = strictMock(Output.class);
-
- output1.init(); expectLastCall();
- output2.init(); expectLastCall();
- output3.init(); expectLastCall();
+
+ LogFeederProps logFeederProps = new LogFeederProps();
+ output1.init(logFeederProps); expectLastCall();
+ output2.init(logFeederProps); expectLastCall();
+ output3.init(logFeederProps); expectLastCall();
replay(output1, output2, output3);
@@ -74,6 +77,7 @@ public class OutputManagerTest {
manager.add(output1);
manager.add(output2);
manager.add(output3);
+ manager.setLogFeederProps(logFeederProps);
manager.init();
@@ -98,11 +102,14 @@ public class OutputManagerTest {
Output output1 = strictMock(Output.class);
Output output2 = strictMock(Output.class);
Output output3 = strictMock(Output.class);
+
+ LogLevelFilterHandler mockFilter = strictMock(LogLevelFilterHandler.class);
expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor);
- expect(mockInput.isUseEventMD5()).andReturn(false);
- expect(mockInput.isGenEventMD5()).andReturn(false);
- expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor);
+ expect(mockInput.isUseEventMD5()).andReturn(false).anyTimes();
+ expect(mockInput.isGenEventMD5()).andReturn(false).anyTimes();
+ expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor).anyTimes();
+ expect(mockFilter.isAllowed(jsonObj, inputMarker)).andReturn(true).anyTimes();
expect(mockInput.getCache()).andReturn(null);
expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
@@ -110,9 +117,10 @@ public class OutputManagerTest {
output2.write(jsonObj, inputMarker); expectLastCall();
output3.write(jsonObj, inputMarker); expectLastCall();
- replay(output1, output2, output3, mockInput);
+ replay(output1, output2, output3, mockFilter, mockInput);
OutputManager manager = new OutputManager();
+ manager.setLogLevelFilterHandler(mockFilter);
manager.add(output1);
manager.add(output2);
manager.add(output3);
@@ -133,17 +141,21 @@ public class OutputManagerTest {
Output output1 = strictMock(Output.class);
Output output2 = strictMock(Output.class);
Output output3 = strictMock(Output.class);
+
+ LogLevelFilterHandler mockFilter = strictMock(LogLevelFilterHandler.class);
- expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor);
+ expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor).anyTimes();
+ expect(mockFilter.isAllowed(jsonString, inputMarker)).andReturn(true).anyTimes();
expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
output1.write(jsonString, inputMarker); expectLastCall();
output2.write(jsonString, inputMarker); expectLastCall();
output3.write(jsonString, inputMarker); expectLastCall();
- replay(output1, output2, output3, mockInput);
+ replay(output1, output2, output3, mockInput, mockFilter);
OutputManager manager = new OutputManager();
+ manager.setLogLevelFilterHandler(mockFilter);
manager.add(output1);
manager.add(output2);
manager.add(output3);
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
index 7c6aca2..78cf014 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.logfeeder.output;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.spool.LogSpooler;
@@ -93,7 +94,7 @@ public class OutputS3FileTest {
}
};
outputS3File.loadConfig(configMap);
- outputS3File.init();
+ outputS3File.init(new LogFeederProps());
outputS3File.write("log event block", inputMarker);
verify(spooler);
}
@@ -130,7 +131,7 @@ public class OutputS3FileTest {
}
};
outputS3File.loadConfig(configMap);
- outputS3File.init();
+ outputS3File.init(new LogFeederProps());
outputS3File.write("log event block1", inputMarker);
outputS3File.write("log event block2", inputMarker);
verify(spooler);
@@ -149,7 +150,7 @@ public class OutputS3FileTest {
OutputS3File outputS3File = new OutputS3File();
configMap.put(S3OutputConfiguration.ROLLOVER_SIZE_THRESHOLD_BYTES_KEY, thresholdSize);
outputS3File.loadConfig(configMap);
- outputS3File.init();
+ outputS3File.init(new LogFeederProps());
assertTrue(outputS3File.shouldRollover(logSpoolerContext));
}
@@ -166,7 +167,7 @@ public class OutputS3FileTest {
OutputS3File outputS3File = new OutputS3File();
configMap.put(S3OutputConfiguration.ROLLOVER_SIZE_THRESHOLD_BYTES_KEY, thresholdSize);
outputS3File.loadConfig(configMap);
- outputS3File.init();
+ outputS3File.init(new LogFeederProps());
assertFalse(outputS3File.shouldRollover(logSpoolerContext));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
index 5ab271a..70d5c8f 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
@@ -26,6 +26,8 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
+import org.apache.ambari.logfeeder.conf.LogFeederSecurityConfig;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
@@ -52,12 +54,16 @@ public class OutputSolrTest {
private OutputSolr outputSolr;
private LogSearchConfigLogFeeder logSearchConfigMock;
private Map<Integer, SolrInputDocument> receivedDocs = new ConcurrentHashMap<>();
+ private LogFeederProps logFeederProps = new LogFeederProps();
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void init() throws Exception {
+ LogFeederSecurityConfig logFeederSecurityConfig = new LogFeederSecurityConfig();
+ logFeederSecurityConfig.setSolrKerberosEnabled(false);
+ logFeederProps.setLogFeederSecurityConfig(logFeederSecurityConfig);
outputSolr = new OutputSolr() {
@SuppressWarnings("deprecation")
@Override
@@ -97,7 +103,7 @@ public class OutputSolrTest {
config.put("type", "service");
outputSolr.loadConfig(config);
- outputSolr.init();
+ outputSolr.init(logFeederProps);
Map<Integer, SolrInputDocument> expectedDocs = new HashMap<>();
@@ -166,7 +172,7 @@ public class OutputSolrTest {
config.put("type", "service");
outputSolr.loadConfig(config);
- outputSolr.init();
+ outputSolr.init(logFeederProps);
}
@After
[35/51] [abbrv] ambari git commit: AMBARI-22680 Ambari web UI does
not fail an invalid repo base URL during cluster installation (dili)
Posted by am...@apache.org.
AMBARI-22680 Ambari web UI does not fail an invalid repo base URL during cluster installation (dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a15fc7fc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a15fc7fc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a15fc7fc
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: a15fc7fc1ac70d3399a1b4046c3124dcc8cdbb11
Parents: 6a37985
Author: Di Li <di...@apache.org>
Authored: Thu Dec 21 15:44:22 2017 -0500
Committer: Di Li <di...@apache.org>
Committed: Thu Dec 21 15:44:22 2017 -0500
----------------------------------------------------------------------
.../internal/RepositoryResourceProvider.java | 14 ++++++++++++--
.../internal/RepositoryResourceProviderTest.java | 2 ++
2 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a15fc7fc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
index edec1ee..60dff69 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
@@ -117,15 +117,25 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
throws SystemException, UnsupportedPropertyException,
NoSuchResourceException, NoSuchParentResourceException {
- final Set<RepositoryRequest> requests = new HashSet<>();
+ final Set<RepositoryRequest> requestsToVerifyBaseURLs = new HashSet<>();
Iterator<Map<String,Object>> iterator = request.getProperties().iterator();
if (iterator.hasNext()) {
for (Map<String, Object> propertyMap : getPropertyMaps(iterator.next(), predicate)) {
- requests.add(getRequest(propertyMap));
+ RepositoryRequest rr = getRequest(propertyMap);
+ if(rr.isVerifyBaseUrl()) {
+ requestsToVerifyBaseURLs.add(rr);
+ }
}
}
+ //Validation only - used by the cluster installation
+ try {
+ getManagementController().verifyRepositories(requestsToVerifyBaseURLs);
+ } catch (AmbariException e) {
+ throw new SystemException("", e);
+ }
+
return getRequestStatus(null);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a15fc7fc/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
index abdef9b..0d7813a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.RepositoryRequest;
import org.apache.ambari.server.controller.RepositoryResponse;
import org.apache.ambari.server.controller.spi.Predicate;
import org.apache.ambari.server.controller.spi.Request;
@@ -176,6 +177,7 @@ public class RepositoryResourceProviderTest {
// set expectations
expect(managementController.getRepositories(EasyMock.anyObject())).andReturn(allResponse).times(1);
+ managementController.verifyRepositories(EasyMock.<Set<RepositoryRequest>>anyObject());
// replay
replay(managementController);
[04/51] [abbrv] ambari git commit: AMBARI-22651 Unable to add/change
role for user. (atkach)
Posted by am...@apache.org.
AMBARI-22651 Unable to add/change role for user. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dc31e516
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dc31e516
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dc31e516
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: dc31e516d95a6636a27d5fbadb9b5529983587c2
Parents: 31e8e55
Author: Andrii Tkach <at...@apache.org>
Authored: Thu Dec 14 12:45:09 2017 +0200
Committer: Andrii Tkach <at...@apache.org>
Committed: Thu Dec 14 12:45:24 2017 +0200
----------------------------------------------------------------------
.../ui/admin-web/app/views/userManagement/groupEdit.html | 2 +-
.../ui/admin-web/app/views/userManagement/modals/groupCreate.html | 2 +-
.../ui/admin-web/app/views/userManagement/modals/userCreate.html | 3 ++-
.../resources/ui/admin-web/app/views/userManagement/userEdit.html | 2 +-
4 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc31e516/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupEdit.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupEdit.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupEdit.html
index 3eca354..4688eb5 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupEdit.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupEdit.html
@@ -50,7 +50,7 @@
<editable-list items-source="group.editingUsers" resource-type="User" editable="group.group_type == 'LOCAL'"></editable-list>
</div>
</div>
- <div class="form-group">
+ <div class="form-group" ng-show="cluster">
<label for="role" class="col-sm-2 roles-label">
{{'groups.role' | translate}}
<i class="fa fa-question-circle" aria-hidden="true" ng-click="showHelpPage()"></i>
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc31e516/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/groupCreate.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/groupCreate.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/groupCreate.html
index 7692a68..8ea1b37 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/groupCreate.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/groupCreate.html
@@ -58,7 +58,7 @@
</div>
</div>
- <div class="row">
+ <div class="row" ng-show="cluster">
<div class="form-group col-sm-6"
ng-class="{ 'has-error': form.groupCreateForm.role.$error.required && form.groupCreateForm.submitted }">
<label for="role" class="nowrap roles-label">
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc31e516/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/userCreate.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/userCreate.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/userCreate.html
index b1b5876..2e5bba6 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/userCreate.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/modals/userCreate.html
@@ -38,6 +38,7 @@
tooltip="{{'users.userNameTip' | translate}}"
autocomplete="off"
tooltip-trigger="focus"
+ tooltip-placement="top"
class="form-control"
ng-model="formData.userName"
name="userName"
@@ -98,7 +99,7 @@
</div>
</div>
- <div class="row">
+ <div class="row" ng-show="cluster">
<div class="form-group col-sm-6"
ng-class="{ 'has-error': form.userCreateForm.role.$error.required && form.userCreateForm.submitted }">
<label for="role" class="nowrap roles-label">
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc31e516/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/userEdit.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/userEdit.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/userEdit.html
index 300046d..80e2196 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/userEdit.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/userEdit.html
@@ -70,7 +70,7 @@
</div>
</div>
- <div class="form-group">
+ <div class="form-group" ng-show="cluster">
<label for="role" class="col-sm-2 roles-label">
{{'users.role' | translate}}
<i class="fa fa-question-circle" aria-hidden="true" ng-click="showHelpPage()"></i>
[26/51] [abbrv] ambari git commit: AMBARI-22682 Ambari 3.0 Admin
View: Add visual-search box to all pages. (atkach)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/UsersListCtrl_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/UsersListCtrl_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/UsersListCtrl_test.js
index fcafa59..db91b56 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/UsersListCtrl_test.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/UsersListCtrl_test.js
@@ -33,312 +33,6 @@ describe('#Cluster', function () {
});
}));
- describe('#clearFilters()', function () {
-
- it('should clear filters and reset pagination', function () {
- scope.currentPage = 2;
- scope.filters.name = 'a';
- scope.filters.status = {
- label: $t('common.local'),
- value: false
- };
- scope.filters.type = {
- label: $t('common.local'),
- value: 'LOCAL'
- };
- scope.clearFilters();
- expect(scope.filters.name).toEqual('');
- expect(scope.filters.status).toEqual({
- label: $t('common.all'),
- value: '*'
- });
- expect(scope.filters.type).toEqual({
- label: $t('common.all'),
- value: '*'
- });
- expect(scope.currentPage).toEqual(1);
- });
-
- });
-
- describe('#isNotEmptyFilter', function () {
-
- var cases = [
- {
- currentNameFilter: '',
- currentTypeFilter: null,
- currentActiveFilter: null,
- isNotEmptyFilter: false,
- title: 'no filters'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: false,
- title: 'empty filters'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name filter'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name filter with "0" as string'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'type filter'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'activity filter'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name and type filters'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'name and activity filters'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name and admin filters'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name and type filters with "0" as string'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'name and activity filters with "0" as string'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name and admin filters with "0" as string'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'type and activity filters'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'type and admin filters'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'activity and admin filters'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'all filters except name one'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'all filters except type one'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'all filters except activity one'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'all filters except admin one'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: '*'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'all filters with "0" as string except type one'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'all filters with "0" as string except activity one'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: 'LOCAL'
- },
- currentActiveFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'all filters with "0" as string except admin one'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: false
- },
- currentActiveFilter: {
- value: 'LOCAL'
- },
- isNotEmptyFilter: true,
- title: 'all filters'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: false
- },
- currentActiveFilter: {
- value: 'LOCAL'
- },
- isNotEmptyFilter: true,
- title: 'all filters with "0" as string'
- }
- ];
-
- cases.forEach(function (item) {
- it(item.title, function () {
- $httpBackend.expectGET(/\/api\/v1\/users/).respond(200);
- scope.filters.name = item.currentNameFilter;
- scope.filters.status = item.currentActiveFilter;
- scope.filters.type = item.currentTypeFilter;
- scope.$digest();
- expect(scope.isNotEmptyFilter).toEqual(item.isNotEmptyFilter);
- });
- });
-
- });
-
});
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
index 9bc7083..0f4e3b3 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
@@ -135,19 +135,6 @@ describe('#comboSearch', function () {
describe('#hideAutocomplete', function() {
- it('showAutoComplete should be false when filter passed', function () {
- var isoScope = element.isolateScope();
- var filter = {
- showAutoComplete: true
- };
- jasmine.Clock.useMock();
-
- isoScope.hideAutocomplete(filter);
-
- jasmine.Clock.tick(101);
- expect(filter.showAutoComplete).toBeFalsy();
- });
-
it('showAutoComplete should be false when isEditing = false', function () {
var isoScope = element.isolateScope();
jasmine.Clock.useMock();
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Filters_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Filters_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Filters_test.js
new file mode 100644
index 0000000..a8b2924
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Filters_test.js
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('Filters Service', function () {
+ var Filters;
+
+ beforeEach(function () {
+ module('ambariAdminConsole', angular.noop);
+ inject(function (_Filters_) {
+ Filters = _Filters_;
+ });
+ });
+ describe('#initFilterOptions', function() {
+ var items = [
+ {
+ k1: {
+ values: ['val1', 'val3']
+ },
+ k2: 'val2'
+ }
+ ];
+ var filters = [
+ {
+ isMultiple: true,
+ key: 'k1',
+ options: [],
+ customValueConverter: function(item) {
+ return item.k1.values;
+ }
+ },
+ {
+ key: 'k2',
+ options: []
+ },
+ {
+ isStatic: true,
+ options: [
+ {
+ key: 'static1',
+ label: 'static1'
+ }
+ ]
+ }
+ ];
+ beforeEach(function() {
+ Filters.initFilterOptions(filters, items);
+ });
+
+ it('should set static options of filters', function() {
+ expect(filters[2].options).toEqual([{
+ key: 'static1',
+ label: 'static1'
+ }]);
+ });
+
+ it('should set options of filters', function() {
+ expect(filters[1].options).toEqual([{
+ key: 'val2',
+ label: 'val2'
+ }]);
+ });
+
+ it('should set multiple options of filters', function() {
+ expect(filters[0].options).toEqual([
+ {
+ key: 'val1',
+ label: 'val1'
+ },
+ {
+ key: 'val3',
+ label: 'val3'
+ }
+ ]);
+ });
+ });
+
+ describe('#filterItems', function() {
+
+ it('all items should be filtered when no filters applied', function() {
+ var items = [{}];
+
+ expect(Filters.filterItems(null, items, [])).toEqual(1);
+ expect(items[0].isFiltered).toBeTruthy();
+ });
+
+ it('items should be filtered when simple filter applied', function() {
+ var appliedFilters = [
+ {
+ key: 'p1',
+ values: ['val1']
+ }
+ ];
+ var items = [
+ { p1: 'val1' },
+ { p1: 'val2' }
+ ];
+ var filterDefinitions = [];
+
+ expect(Filters.filterItems(appliedFilters, items, filterDefinitions)).toEqual(1);
+ expect(items[0].isFiltered).toBeTruthy();
+ expect(items[1].isFiltered).toBeFalsy();
+ });
+
+ it('items should be filtered when filter applied on array values', function() {
+ var appliedFilters = [
+ {
+ key: 'p1',
+ values: ['a']
+ }
+ ];
+ var items = [
+ { p1: ['a', 'b'] },
+ { p1: ['c', 'b'] }
+ ];
+ var filterDefinitions = [];
+
+ expect(Filters.filterItems(appliedFilters, items, filterDefinitions)).toEqual(1);
+ expect(items[0].isFiltered).toBeTruthy();
+ expect(items[1].isFiltered).toBeFalsy();
+ });
+
+ it('items should be filtered when custom filter applied', function() {
+ var appliedFilters = [
+ {
+ key: 'p1',
+ values: ['a']
+ }
+ ];
+ var items = [
+ { p1: { customValue: 'a' } },
+ { p1: { customValue: 'b' } }
+ ];
+ var filterDefinitions = [{
+ key: 'p1',
+ customValueConverter: function(item) {
+ return item.p1.customValue;
+ }
+ }];
+
+ expect(Filters.filterItems(appliedFilters, items, filterDefinitions)).toEqual(1);
+ expect(items[0].isFiltered).toBeTruthy();
+ expect(items[1].isFiltered).toBeFalsy();
+ });
+ });
+
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Pagination_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Pagination_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Pagination_test.js
new file mode 100644
index 0000000..ce6cb52
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Pagination_test.js
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('Pagination Service', function () {
+ var Pagination, $pagination;
+
+ beforeEach(function () {
+ module('ambariAdminConsole', angular.noop);
+ inject(function (_Pagination_) {
+ Pagination = _Pagination_;
+ $pagination = Pagination.create({
+ itemsPerPage: 1
+ });
+ });
+ });
+
+ describe('#pageChanged', function() {
+
+ it('should show items on second page', function() {
+ var items = [
+ { isFiltered: true },
+ { isFiltered: true },
+ { isFiltered: true }
+ ];
+ var tableInfo = {
+ showed: 0
+ };
+ $pagination.currentPage = 2;
+ $pagination.pageChanged(items, tableInfo);
+ expect(items[0].isShowed).toBeFalsy();
+ expect(items[1].isShowed).toBeTruthy();
+ expect(items[2].isShowed).toBeFalsy();
+ expect(tableInfo.showed).toEqual(1);
+ });
+ });
+
+ describe('#resetPagination', function() {
+
+ it('should show items on first page', function() {
+ var items = [
+ { isFiltered: true },
+ { isFiltered: true },
+ { isFiltered: true }
+ ];
+ var tableInfo = {
+ showed: 0
+ };
+ $pagination.currentPage = 2;
+ $pagination.resetPagination(items, tableInfo);
+ expect(items[0].isShowed).toBeTruthy();
+ expect(items[1].isShowed).toBeFalsy();
+ expect(items[2].isShowed).toBeFalsy();
+ expect(tableInfo.showed).toEqual(1);
+ });
+ });
+
+});
[16/51] [abbrv] ambari git commit: AMBARI-22522 - Livy server fails
to start during downgrade due to absence of 'conf' directory (Dmitro
Lisnichenko via jonathanhurley)
Posted by am...@apache.org.
AMBARI-22522 - Livy server fails to start during downgrade due to absence of 'conf' directory (Dmitro Lisnichenko via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2817ad48
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2817ad48
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2817ad48
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 2817ad481507033bccd1732943c305108585dc4b
Parents: fded822
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Sun Dec 17 20:48:31 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Dec 18 15:53:16 2017 -0500
----------------------------------------------------------------------
.../custom_actions/scripts/install_packages.py | 37 +++++++++++++++++++-
1 file changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2817ad48/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index 862f205..c3fbc48 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -31,7 +31,6 @@ from resource_management.core.resources import Package
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.stack_select import get_stack_versions
-from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.repo_version_history \
import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE
from resource_management.core.providers import get_provider
@@ -155,11 +154,47 @@ class InstallPackages(Script):
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
+ self._fix_default_links("livy", "livy-server")
+ self._fix_default_links("livy2", "livy2-server")
+
# if installing a version of HDP that needs some symlink love, then create them
if is_package_install_successful and 'actual_version' in self.structured_output:
self._relink_configurations_with_conf_select(stack_id, self.structured_output['actual_version'])
+ def _fix_default_links(self, package_name, component_name):
+ """
+ If a prior version of Ambari did not correctly reverse the conf symlinks, then they would
+ be put into a bad state when distributing a new stack. For example:
+
+ /etc/component/conf (directory)
+ <stack-root>/v1/component/conf -> /etc/component/conf
+
+ When distributing v2, we'd detect the /etc/component/conf problems and would try to adjust it:
+ /etc/component/conf -> <stack-root>/current/component/conf
+ <stack-root>/v2/component/conf -> /etc/component/v2/0
+
+ The problem is that v1 never gets changed (since the stack being distributed is v2), and
+ we end up with a circular link:
+ /etc/component/conf -> <stack-root>/current/component/conf
+ <stack-root>/v1/component/conf -> /etc/component/conf
+
+ :return: None
+ """
+ from resource_management.libraries.functions import stack_select
+ package_dirs = conf_select.get_package_dirs()
+ if package_name in package_dirs:
+ Logger.info("Attempting to fix the default conf links for {0}".format(package_name))
+
+ directories = package_dirs[package_name]
+ Logger.info("The following directories will be fixed for {0}: {1}".format(package_name,
+ str(directories)))
+
+ stack_version = stack_select.get_stack_version_before_install(component_name)
+ if stack_version:
+ conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories)
+
+
def _relink_configurations_with_conf_select(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
[10/51] [abbrv] ambari git commit: AMBARI-22663 Log Search UI:
incorrect caption for graph gap in weeks. (ababiichuk)
Posted by am...@apache.org.
AMBARI-22663 Log Search UI: incorrect caption for graph gap in weeks. (ababiichuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f9ad13d0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f9ad13d0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f9ad13d0
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: f9ad13d0a304414b7e37b6a61afac876912e74ce
Parents: ab342d7
Author: ababiichuk <ab...@hortonworks.com>
Authored: Mon Dec 18 13:30:27 2017 +0200
Committer: ababiichuk <ab...@hortonworks.com>
Committed: Mon Dec 18 13:30:27 2017 +0200
----------------------------------------------------------------------
ambari-logsearch/ambari-logsearch-web/src/assets/i18n/en.json | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9ad13d0/ambari-logsearch/ambari-logsearch-web/src/assets/i18n/en.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/assets/i18n/en.json b/ambari-logsearch/ambari-logsearch-web/src/assets/i18n/en.json
index 3f4f5c8..9561ca0 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/assets/i18n/en.json
+++ b/ambari-logsearch/ambari-logsearch-web/src/assets/i18n/en.json
@@ -169,5 +169,7 @@
"histogram.gap.hour": "hour",
"histogram.gap.hours": "hours",
"histogram.gap.day": "day",
- "histogram.gap.days": "days"
+ "histogram.gap.days": "days",
+ "histogram.gap.week": "week",
+ "histogram.gap.weeks": "weeks"
}
[15/51] [abbrv] ambari git commit: AMBARI-22655 - Livy/Livy2 Unable
To Start Due to Address Already In Use (jonathanhurley)
Posted by am...@apache.org.
AMBARI-22655 - Livy/Livy2 Unable To Start Due to Address Already In Use (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fded8228
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fded8228
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fded8228
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: fded8228015231afcae0900502db27ddc863773a
Parents: 1d87b21
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Dec 14 21:59:19 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Dec 18 15:47:45 2017 -0500
----------------------------------------------------------------------
.../SPARK/1.2.1/package/scripts/livy_service.py | 22 +++++++++------
.../SPARK/2.2.0/package/scripts/livy_service.py | 29 +++++++++-----------
.../2.0.0/package/scripts/livy2_service.py | 27 ++++++++----------
.../SPARK2/2.0.0/package/scripts/setup_livy2.py | 2 ++
.../HDP/2.0.6/properties/stack_packages.json | 8 +++++-
.../HDP/3.0/properties/stack_packages.json | 8 +++++-
.../python/stacks/2.5/SPARK/test_spark_livy.py | 3 +-
.../stacks/2.6/SPARK2/test_spark_livy2.py | 5 +++-
8 files changed, 60 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
index cce2148..add5a77 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
@@ -21,28 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
- 'ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy_server_pid_file}` >/dev/null 2>&1')
+ # use the livy user to get the PID (it is protected on non-root systems)
+ livy_server_pid = get_user_call_output.get_user_call_output(format("cat {livy_server_pid_file}"),
+ user=params.livy_user, is_checked_call=False)[1]
+
+ livy_server_pid = livy_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy_server_start}'),
user=params.livy_user,
environment={'JAVA_HOME': params.java_home},
- not_if=livyserver_no_op_test)
+ not_if=process_id_exists_command
+ )
elif action == 'stop':
- pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy_server_pid_file} || cat {livy_server_pid_file}"), user=params.livy_user)[1]
- pid = pid.replace("\n", " ")
Execute(format('{livy_server_stop}'),
user=params.livy_user,
- only_if=livyserver_no_op_test,
+ only_if=process_id_exists_command,
timeout=10,
- on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E kill -9 {pid}"),
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
+
File(params.livy_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
index 273b056..add5a77 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
@@ -21,35 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
- 'ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy_server_pid_file}` >/dev/null 2>&1')
+ # use the livy user to get the PID (it is protected on non-root systems)
+ livy_server_pid = get_user_call_output.get_user_call_output(format("cat {livy_server_pid_file}"),
+ user=params.livy_user, is_checked_call=False)[1]
+
+ livy_server_pid = livy_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy_server_start}'),
user=params.livy_user,
environment={'JAVA_HOME': params.java_home},
- not_if=livyserver_no_op_test
- )
+ not_if=process_id_exists_command
+ )
elif action == 'stop':
- pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy_server_pid_file} || cat {livy_server_pid_file}"), user=params.livy_user)[1]
- pid = pid.replace("\n", " ")
Execute(format('{livy_server_stop}'),
user=params.livy_user,
- only_if=livyserver_no_op_test,
+ only_if=process_id_exists_command,
timeout=10,
- on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E kill -9 {pid}"),
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
- File(params.livy_server_pid_file,
- action="delete"
- )
-
-
-
+ File(params.livy_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
index dfadd84..0180a31 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
@@ -21,35 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy2_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
- 'ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy2_server_pid_file}` >/dev/null 2>&1')
+ # use the livy2 user to get the PID (it is protected on non-root systems)
+ livy2_server_pid = get_user_call_output.get_user_call_output(format("cat {livy2_server_pid_file}"),
+ user=params.livy2_user, is_checked_call=False)[1]
+
+ livy2_server_pid = livy2_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p {livy2_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy2_server_start}'),
user=params.livy2_user,
environment={'JAVA_HOME': params.java_home},
- not_if=livyserver_no_op_test
+ not_if=process_id_exists_command
)
elif action == 'stop':
- pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy2_server_pid_file} || cat {livy2_server_pid_file}"), user=params.livy2_user)[1]
- pid = pid.replace("\n", " ")
Execute(format('{livy2_server_stop}'),
user=params.livy2_user,
- only_if=livyserver_no_op_test,
+ only_if=process_id_exists_command,
timeout=10,
- on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E kill -9 {pid}"),
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy2_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
- File(params.livy2_server_pid_file,
- action="delete"
- )
-
-
-
+ File(params.livy2_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
index c428488..59ec190 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
@@ -29,8 +29,10 @@ def setup_livy(env, type, upgrade_type = None, action = None):
owner=params.livy2_user,
group=params.user_group,
mode=0775,
+ cd_access = 'a',
create_parents = True
)
+
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
index 01a30f3..6a0eadd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
@@ -1067,6 +1067,12 @@
"current_dir": "{0}/current/livy-client/conf"
}
],
+ "livy2": [
+ {
+ "conf_dir": "/etc/livy2/conf",
+ "current_dir": "{0}/current/livy2-client/conf"
+ }
+ ],
"mahout": [
{
"conf_dir": "/etc/mahout/conf",
@@ -1241,7 +1247,7 @@
"packages": ["spark", "livy"]
},
"SPARK2": {
- "packages": ["spark2", "livy"]
+ "packages": ["spark2", "livy2"]
},
"SQOOP": {
"packages": ["sqoop"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
index 28efbfa..63fcf4b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
@@ -944,6 +944,12 @@
"current_dir": "{0}/current/livy-client/conf"
}
],
+ "livy2": [
+ {
+ "conf_dir": "/etc/livy2/conf",
+ "current_dir": "{0}/current/livy2-client/conf"
+ }
+ ],
"mahout": [
{
"conf_dir": "/etc/mahout/conf",
@@ -1118,7 +1124,7 @@
"packages": ["spark", "livy"]
},
"SPARK2": {
- "packages": ["spark2", "livy"]
+ "packages": ["spark2", "livy2"]
},
"SQOOP": {
"packages": ["sqoop"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
index 4f1fb17..e37c6fa 100644
--- a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
+++ b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
@@ -25,6 +25,7 @@ from only_for_platform import not_for_platform, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
@patch("resource_management.libraries.functions.get_stack_version", new=MagicMock(return_value="2.5.0.0-1597"))
+@patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0,'12345','')))
class TestSparkClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.1/package"
STACK_VERSION = "2.5"
@@ -146,6 +147,6 @@ class TestSparkClient(RMFTestCase):
)
self.assertResourceCalled('Execute', '/usr/hdp/current/livy-server/bin/livy-server start',
environment = {'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'},
- not_if = 'ls /var/run/livy/livy-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy/livy-livy-server.pid` >/dev/null 2>&1',
+ not_if = 'ls /var/run/livy/livy-livy-server.pid >/dev/null 2>&1 && ps -p 12345 >/dev/null 2>&1',
user = 'livy'
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fded8228/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
index 60e7fd2..f0a173b 100644
--- a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
+++ b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
@@ -25,6 +25,7 @@ from only_for_platform import not_for_platform, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
@patch("resource_management.libraries.functions.get_stack_version", new=MagicMock(return_value="2.5.0.0-1597"))
+@patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0,'12345','')))
class TestSparkClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK2/2.0.0/package"
STACK_VERSION = "2.6"
@@ -45,12 +46,14 @@ class TestSparkClient(RMFTestCase):
owner = 'livy',
group = 'hadoop',
create_parents = True,
+ cd_access = 'a',
mode = 0775
)
self.assertResourceCalled('Directory', '/var/log/livy2',
owner = 'livy',
group = 'hadoop',
create_parents = True,
+ cd_access = 'a',
mode = 0775
)
self.assertResourceCalled('HdfsResource', '/user/livy',
@@ -146,6 +149,6 @@ class TestSparkClient(RMFTestCase):
)
self.assertResourceCalled('Execute', '/usr/hdp/current/livy2-server/bin/livy-server start',
environment = {'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'},
- not_if = 'ls /var/run/livy2/livy-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy2/livy-livy-server.pid` >/dev/null 2>&1',
+ not_if = 'ls /var/run/livy2/livy-livy-server.pid >/dev/null 2>&1 && ps -p 12345 >/dev/null 2>&1',
user = 'livy'
)
[30/51] [abbrv] ambari git commit: AMBARI-22530. Refactor internal
code of handling info between kerberos wizard actions (echekanskiy)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
index 52ab9b5..d90d5bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
@@ -36,9 +36,10 @@ import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
import org.apache.ambari.server.orm.dao.HostDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor;
@@ -101,12 +102,6 @@ public class HostKerberosIdentityResourceProvider extends ReadOnlyResourceProvid
private KerberosHelper kerberosHelper;
/**
- * KerberosPrincipalHostDAO used to get Kerberos principal details
- */
- @Inject
- private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
-
- /**
* KerberosPrincipalDAO used to get Kerberos principal details
*/
@Inject
@@ -118,6 +113,9 @@ public class HostKerberosIdentityResourceProvider extends ReadOnlyResourceProvid
@Inject
private HostDAO hostDAO;
+ @Inject
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
/**
* Create a new resource provider for the given management controller.
*
@@ -200,7 +198,8 @@ public class HostKerberosIdentityResourceProvider extends ReadOnlyResourceProvid
if ((hostId != null) && kerberosPrincipalDAO.exists(principal)) {
if (keytabDescriptor != null) {
- if (kerberosPrincipalHostDAO.exists(principal, hostId, keytabDescriptor.getFile())) {
+ KerberosKeytabPrincipalEntity entity = kerberosKeytabPrincipalDAO.findByNaturalKey(hostId, keytabDescriptor.getFile(), principal);
+ if (entity != null && entity.isDistributed()) {
installedStatus = "true";
} else {
installedStatus = "false";
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
index a8723b7..ca7d23c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
@@ -18,14 +18,13 @@
package org.apache.ambari.server.orm.dao;
-import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import org.apache.ambari.server.orm.RequiresSession;
-import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
import com.google.inject.Inject;
@@ -35,76 +34,103 @@ import com.google.inject.persist.Transactional;
@Singleton
public class KerberosKeytabDAO {
- @Inject
- Provider<EntityManager> entityManagerProvider;
-
- @Transactional
- public void create(KerberosKeytabEntity kerberosKeytabEntity) {
- entityManagerProvider.get().persist(kerberosKeytabEntity);
- }
-
- public void create(String keytabPath) {
- create(new KerberosKeytabEntity(keytabPath));
- }
-
- @Transactional
- public KerberosKeytabEntity merge(KerberosKeytabEntity kerberosKeytabEntity) {
- return entityManagerProvider.get().merge(kerberosKeytabEntity);
- }
-
- @Transactional
- public void remove(KerberosKeytabEntity kerberosKeytabEntity) {
- entityManagerProvider.get().remove(merge(kerberosKeytabEntity));
- }
-
- public void remove(String keytabPath) {
- KerberosKeytabEntity kke = find(keytabPath);
- if (kke != null) {
- remove(kke);
- }
- }
-
- @Transactional
- public void refresh(KerberosKeytabEntity kerberosKeytabEntity) {
- entityManagerProvider.get().refresh(kerberosKeytabEntity);
+ @Inject
+ Provider<EntityManager> entityManagerProvider;
+
+ @Inject
+ KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
+ @Transactional
+ public void create(KerberosKeytabEntity kerberosKeytabEntity) {
+ entityManagerProvider.get().persist(kerberosKeytabEntity);
+ }
+
+ public void create(String keytabPath) {
+ create(new KerberosKeytabEntity(keytabPath));
+ }
+
+ @Transactional
+ public KerberosKeytabEntity merge(KerberosKeytabEntity kerberosKeytabEntity) {
+ return entityManagerProvider.get().merge(kerberosKeytabEntity);
+ }
+
+ @Transactional
+ public void remove(KerberosKeytabEntity kerberosKeytabEntity) {
+ entityManagerProvider.get().remove(merge(kerberosKeytabEntity));
+ }
+
+ public void remove(String keytabPath) {
+ KerberosKeytabEntity kke = find(keytabPath);
+ if (kke != null) {
+ remove(kke);
}
+ }
+ @Transactional
+ public void refresh(KerberosKeytabEntity kerberosKeytabEntity) {
+ entityManagerProvider.get().refresh(kerberosKeytabEntity);
+ }
- @RequiresSession
- public KerberosKeytabEntity find(String keytabPath) {
- return entityManagerProvider.get().find(KerberosKeytabEntity.class, keytabPath);
- }
- @RequiresSession
- public List<KerberosKeytabEntity> findAll() {
- TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
- createNamedQuery("KerberosKeytabEntity.findAll", KerberosKeytabEntity.class);
+ @RequiresSession
+ public KerberosKeytabEntity find(String keytabPath) {
+ return entityManagerProvider.get().find(KerberosKeytabEntity.class, keytabPath);
+ }
- return query.getResultList();
+ @RequiresSession
+ public List<KerberosKeytabEntity> findByPrincipalAndHost(String principalName, Long hostId) {
+ if(hostId == null) {
+ return findByPrincipalAndNullHost(principalName);
}
-
- @RequiresSession
- public boolean exists(String keytabPath) {
- return find(keytabPath) != null;
+ TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabEntity.findByPrincipalAndHost", KerberosKeytabEntity.class);
+ query.setParameter("hostId", hostId);
+ query.setParameter("principalName", principalName);
+ List<KerberosKeytabEntity> result = query.getResultList();
+ if(result == null) {
+ return Collections.emptyList();
}
-
- @RequiresSession
- public Collection<KerberosKeytabEntity> findByHost(Long hostId) {
- TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
- createNamedQuery("KerberosKeytabEntity.findByHost", KerberosKeytabEntity.class);
- query.setParameter("hostId", hostId);
- return query.getResultList();
+ return result;
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabEntity> findByPrincipalAndNullHost(String principalName) {
+ TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabEntity.findByPrincipalAndNullHost", KerberosKeytabEntity.class);
+ query.setParameter("principalName", principalName);
+ List<KerberosKeytabEntity> result = query.getResultList();
+ if(result == null) {
+ return Collections.emptyList();
}
-
- public Collection<KerberosKeytabEntity> findByHost(HostEntity hostEntity) {
- return findByHost(hostEntity.getHostId());
+ return result;
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabEntity> findAll() {
+ TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabEntity.findAll", KerberosKeytabEntity.class);
+ List<KerberosKeytabEntity> result = query.getResultList();
+ if(result == null) {
+ return Collections.emptyList();
}
-
- public void remove(List<KerberosKeytabEntity> entities) {
- if (entities != null) {
- for (KerberosKeytabEntity entity : entities) {
- entityManagerProvider.get().remove(entity);
- }
- }
+ return result;
+ }
+
+ @RequiresSession
+ public boolean exists(String keytabPath) {
+ return find(keytabPath) != null;
+ }
+
+ @RequiresSession
+ public boolean exists(KerberosKeytabEntity kerberosKeytabEntity) {
+ return find(kerberosKeytabEntity.getKeytabPath()) != null;
+ }
+
+ public void remove(List<KerberosKeytabEntity> entities) {
+ if (entities != null) {
+ for (KerberosKeytabEntity entity : entities) {
+ remove(entity);
+ }
}
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java
new file mode 100644
index 0000000..bf4b75b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.orm.dao;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import javax.persistence.EntityManager;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaQuery;
+import javax.persistence.criteria.Join;
+import javax.persistence.criteria.Predicate;
+import javax.persistence.criteria.Root;
+
+import org.apache.ambari.server.orm.RequiresSession;
+import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabServiceMappingEntity;
+import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.google.inject.persist.Transactional;
+
+@Singleton
+public class KerberosKeytabPrincipalDAO {
+ @Inject
+ Provider<EntityManager> entityManagerProvider;
+
+ @Inject
+ HostDAO hostDAO;
+
+ @Transactional
+ public void create(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity) {
+ entityManagerProvider.get().persist(kerberosKeytabPrincipalEntity);
+ }
+
+ @Transactional
+ public void create(
+ KerberosKeytabEntity kerberosKeytabEntity,
+ HostEntity hostEntity,
+ KerberosPrincipalEntity principalEntity) {
+ entityManagerProvider.get().persist(
+ new KerberosKeytabPrincipalEntity(kerberosKeytabEntity, hostEntity, principalEntity)
+ );
+ }
+
+ /**
+ * Find or create {@link KerberosKeytabPrincipalEntity} with specified dependecies.
+ *
+ * @param kerberosKeytabEntity {@link KerberosKeytabEntity} which owns this principal
+ * @param hostEntity {@link HostEntity} which owns this principal
+ * @param principalEntity {@link KerberosPrincipalEntity} which related to this principal
+ * @return evaluated entity
+ */
+ public KerberosKeytabPrincipalEntity findOrCreate(KerberosKeytabEntity kerberosKeytabEntity, HostEntity hostEntity, KerberosPrincipalEntity principalEntity)
+ {
+ Long hostId = hostEntity == null ? null : hostEntity.getHostId();
+ KerberosKeytabPrincipalEntity kkp = findByNaturalKey(hostId, kerberosKeytabEntity.getKeytabPath(), principalEntity.getPrincipalName());
+ if (kkp == null) {
+ kkp = new KerberosKeytabPrincipalEntity(
+ kerberosKeytabEntity,
+ hostEntity,
+ principalEntity
+ );
+ create(kkp);
+ kerberosKeytabEntity.addKerberosKeytabPrincipal(kkp);
+ }
+ return kkp;
+ }
+
+ @Transactional
+ public KerberosKeytabPrincipalEntity merge(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity) {
+ return entityManagerProvider.get().merge(kerberosKeytabPrincipalEntity);
+ }
+
+ @Transactional
+ public void remove(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity) {
+ entityManagerProvider.get().remove(merge(kerberosKeytabPrincipalEntity));
+ }
+
+ public void remove(Collection<KerberosKeytabPrincipalEntity> kerberosKeytabPrincipalEntities) {
+ for (KerberosKeytabPrincipalEntity entity : kerberosKeytabPrincipalEntities) {
+ remove(entity);
+ }
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabPrincipalEntity> findByPrincipal(String principal) {
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabPrincipalEntity.findByPrincipal", KerberosKeytabPrincipalEntity.class);
+ query.setParameter("principalName", principal);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null) {
+ return Collections.emptyList();
+ }
+ return result;
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabPrincipalEntity> findByHost(Long hostId) {
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabPrincipalEntity.findByHost", KerberosKeytabPrincipalEntity.class);
+ query.setParameter("hostId", hostId);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null) {
+ return Collections.emptyList();
+ }
+ return result;
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabPrincipalEntity> findByHostAndKeytab(Long hostId, String keytabPath) {
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabPrincipalEntity.findByHostAndKeytab", KerberosKeytabPrincipalEntity.class);
+ query.setParameter("hostId", hostId);
+ query.setParameter("keytabPath", keytabPath);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null) {
+ return Collections.emptyList();
+ }
+ return result;
+ }
+
+ @RequiresSession
+ public KerberosKeytabPrincipalEntity findByHostKeytabAndPrincipal(Long hostId, String keytabPath, String principalName) {
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabPrincipalEntity.findByHostKeytabAndPrincipal", KerberosKeytabPrincipalEntity.class);
+ query.setParameter("hostId", hostId);
+ query.setParameter("keytabPath", keytabPath);
+ query.setParameter("principalName", principalName);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null || result.size() == 0) {
+ return null;
+ } else {
+ return result.get(0);
+ }
+ }
+
+ @RequiresSession
+ public KerberosKeytabPrincipalEntity findByKeytabAndPrincipalNullHost(String keytabPath, String principal) {
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabPrincipalEntity.findByKeytabAndPrincipalNullHost", KerberosKeytabPrincipalEntity.class);
+ query.setParameter("keytabPath", keytabPath);
+ query.setParameter("principalName", principal);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null || result.size() == 0) {
+ return null;
+ } else {
+ return result.get(0);
+ }
+ }
+
+ /**
+ * Ideally for this record PK must be (hostId, keytabPath, principalName), but in some cases hostId can be null.
+ * So surrogate auto-generated PK used, and unique constraint for (hostId, keytabPath, principalName) applied.
+ * This method checks if hostId is null and calls specific method.
+ *
+ * @param hostId host id
+ * @param keytabPath keytab path
+ * @param principalName principal name
+ * @return keytab found
+ */
+ public KerberosKeytabPrincipalEntity findByNaturalKey(Long hostId, String keytabPath, String principalName) {
+ if (hostId == null) {
+ return findByKeytabAndPrincipalNullHost(keytabPath, principalName);
+ } else {
+ return findByHostKeytabAndPrincipal(hostId, keytabPath, principalName);
+ }
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabPrincipalEntity> findByFilter(KerberosKeytabPrincipalFilter filter) {
+ CriteriaBuilder cb = entityManagerProvider.get().getCriteriaBuilder();
+ CriteriaQuery<KerberosKeytabPrincipalEntity> cq = cb.createQuery(KerberosKeytabPrincipalEntity.class);
+ Root<KerberosKeytabPrincipalEntity> root = cq.from(KerberosKeytabPrincipalEntity.class);
+ ArrayList<Predicate> predicates = new ArrayList<>();
+ if (filter.getServiceNames() != null && filter.getServiceNames().size() > 0)
+ {
+ Join<KerberosKeytabPrincipalEntity, KerberosKeytabServiceMappingEntity> mappingJoin = root.join("serviceMapping");
+ predicates.add(mappingJoin.get("serviceName").in(filter.getServiceNames()));
+ if (filter.getComponentNames() != null && filter.getComponentNames().size() > 0) {
+ predicates.add(mappingJoin.get("componentName").in(filter.getComponentNames()));
+ }
+ }
+ if (filter.getHostNames() != null && filter.getHostNames().size() > 0) {
+ List<Long> hostIds = new ArrayList<>();
+ for (String hostname : filter.getHostNames()) {
+ hostIds.add(hostDAO.findByName(hostname).getHostId());
+ }
+ predicates.add(root.get("hostId").in(hostIds));
+ }
+ if (filter.getPrincipals() != null && filter.getPrincipals().size() > 0) {
+ predicates.add(root.get("principalName").in(filter.getPrincipals()));
+ }
+ cq.where(cb.and(predicates.toArray(new Predicate[predicates.size()])));
+
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().createQuery(cq);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null) {
+ return Collections.emptyList();
+ }
+ return result;
+ }
+
+
+ public List<KerberosKeytabPrincipalEntity> findByFilters(Collection<KerberosKeytabPrincipalFilter> filters) {
+ ArrayList<KerberosKeytabPrincipalEntity> result = new ArrayList<>();
+ for (KerberosKeytabPrincipalFilter filter : filters) {
+ result.addAll(findByFilter(filter));
+ }
+ return result;
+ }
+
+ @RequiresSession
+ public boolean exists(Long hostId, String keytabPath, String principalName) {
+ return findByNaturalKey(hostId, keytabPath, principalName) != null;
+ }
+
+ @RequiresSession
+ public List<KerberosKeytabPrincipalEntity> findAll() {
+ TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
+ createNamedQuery("KerberosKeytabPrincipalEntity.findAll", KerberosKeytabPrincipalEntity.class);
+ List<KerberosKeytabPrincipalEntity> result = query.getResultList();
+ if (result == null) {
+ return Collections.emptyList();
+ }
+ return result;
+ }
+
+ @Transactional
+ public void remove(List<KerberosKeytabPrincipalEntity> entities) {
+ if (entities != null) {
+ for (KerberosKeytabPrincipalEntity entity : entities) {
+ entityManagerProvider.get().remove(merge(entity));
+ }
+ }
+ }
+
+ public void removeByHost(Long hostId) {
+ remove(findByHost(hostId));
+ }
+
+ public static class KerberosKeytabPrincipalFilter {
+ private Collection<String> hostNames;
+ private Collection<String> serviceNames;
+ private Collection<String> componentNames;
+ private Collection<String> principals;
+
+ public KerberosKeytabPrincipalFilter(Collection<String> hostNames, Collection<String> serviceNames, Collection<String> componentNames, Collection<String> principals) {
+ this.hostNames = hostNames;
+ this.serviceNames = serviceNames;
+ this.componentNames = componentNames;
+ this.principals = principals;
+ }
+
+ public Collection<String> getHostNames() {
+ return hostNames;
+ }
+
+ public void setHostNames(Collection<String> hostNames) {
+ this.hostNames = hostNames;
+ }
+
+ public Collection<String> getServiceNames() {
+ return serviceNames;
+ }
+
+ public void setServiceNames(Collection<String> serviceNames) {
+ this.serviceNames = serviceNames;
+ }
+
+ public Collection<String> getComponentNames() {
+ return componentNames;
+ }
+
+ public void setComponentNames(Collection<String> componentNames) {
+ this.componentNames = componentNames;
+ }
+
+ public Collection<String> getPrincipals() {
+ return principals;
+ }
+
+ public void setPrincipals(Collection<String> principals) {
+ this.principals = principals;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
index 81e4b3d..5367e9b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
@@ -46,12 +46,6 @@ public class KerberosPrincipalDAO {
Provider<EntityManager> entityManagerProvider;
/**
- * Kerberos Principal Host DAO
- */
- @Inject
- private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
-
- /**
* Make an instance managed and persistent.
*
* @param kerberosPrincipalEntity entity to persist
@@ -95,9 +89,6 @@ public class KerberosPrincipalDAO {
EntityManager entityManager = entityManagerProvider.get();
String principalName = kerberosPrincipalEntity.getPrincipalName();
- // Remove child entities...
- kerberosPrincipalHostDAO.removeByPrincipal(principalName);
-
kerberosPrincipalEntity = find(principalName);
if (kerberosPrincipalEntity != null) {
entityManager.remove(kerberosPrincipalEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
deleted file mode 100644
index f27dc48..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.orm.dao;
-
-
-import java.util.List;
-
-import javax.persistence.EntityManager;
-import javax.persistence.TypedQuery;
-
-import org.apache.ambari.server.orm.RequiresSession;
-import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
-import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntityPK;
-
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-import com.google.inject.persist.Transactional;
-
-
-/**
- * HostKerberosPrincipal Data Access Object.
- */
-@Singleton
-public class KerberosPrincipalHostDAO {
-
- /**
- * JPA entity manager
- */
- @Inject
- Provider<EntityManager> entityManagerProvider;
-
- /**
- * Make an instance managed and persistent.
- *
- * @param kerberosPrincipalHostEntity entity to persist
- */
- @Transactional
- public void create(KerberosPrincipalHostEntity kerberosPrincipalHostEntity) {
- entityManagerProvider.get().persist(kerberosPrincipalHostEntity);
- }
-
- public void create(String principal, Long hostId, String keytabPath) {
- create(new KerberosPrincipalHostEntity(principal, hostId, keytabPath));
- }
-
- /**
- * Merge the state of the given entity into the current persistence context.
- *
- * @param kerberosPrincipalHostEntity entity to merge
- * @return the merged entity
- */
- @Transactional
- public KerberosPrincipalHostEntity merge(KerberosPrincipalHostEntity kerberosPrincipalHostEntity) {
- return entityManagerProvider.get().merge(kerberosPrincipalHostEntity);
- }
-
- /**
- * Remove the entity instance.
- *
- * @param kerberosPrincipalHostEntity entity to remove
- */
- @Transactional
- public void remove(KerberosPrincipalHostEntity kerberosPrincipalHostEntity) {
- entityManagerProvider.get().remove(merge(kerberosPrincipalHostEntity));
- }
-
- /**
- * Refresh the state of the instance from the database,
- * overwriting changes made to the entity, if any.
- *
- * @param kerberosPrincipalHostEntity entity to refresh
- */
- @Transactional
- public void refresh(KerberosPrincipalHostEntity kerberosPrincipalHostEntity) {
- entityManagerProvider.get().refresh(kerberosPrincipalHostEntity);
- }
-
- /**
- * Finds KerberosPrincipalHostEntities for the requested principal
- *
- * @param principalName a String indicating the name of the requested principal
- * @return a List of requested KerberosPrincipalHostEntities or null if none were found
- */
- @RequiresSession
- public List<KerberosPrincipalHostEntity> findByPrincipal(String principalName) {
- final TypedQuery<KerberosPrincipalHostEntity> query = entityManagerProvider.get()
- .createNamedQuery("KerberosPrincipalHostEntityFindByPrincipal", KerberosPrincipalHostEntity.class);
- query.setParameter("principalName", principalName);
- return query.getResultList();
- }
-
- /**
- * Find KerberosPrincipalHostEntities for the requested host
- *
- * @param hostId a Long indicating the id of the requested host
- * @return a List of requested KerberosPrincipalHostEntities or null if none were found
- */
- @RequiresSession
- public List<KerberosPrincipalHostEntity> findByHost(Long hostId) {
- final TypedQuery<KerberosPrincipalHostEntity> query = entityManagerProvider.get()
- .createNamedQuery("KerberosPrincipalHostEntityFindByHost", KerberosPrincipalHostEntity.class);
- query.setParameter("hostId", hostId);
- return query.getResultList();
- }
-
- /**
- * Find KerberosPrincipalHostEntities for the requested host
- *
- * @return a List of requested KerberosPrincipalHostEntities or null if none were found
- */
- @RequiresSession
- public List<KerberosPrincipalHostEntity> findByKeytabPath(String keytabPath) {
- final TypedQuery<KerberosPrincipalHostEntity> query = entityManagerProvider.get()
- .createNamedQuery("KerberosPrincipalHostEntityFindByKeytabPath", KerberosPrincipalHostEntity.class);
- query.setParameter("keytabPath", keytabPath);
- return query.getResultList();
- }
-
- /**
- * Find the KerberosPrincipalHostEntity for the specified primary key
- *
- * @param primaryKey a KerberosPrincipalHostEntityPK containing the requested principal and host names
- * @return the KerberosPrincipalHostEntity or null if not found
- */
- @RequiresSession
- public KerberosPrincipalHostEntity find(KerberosPrincipalHostEntityPK primaryKey) {
- return entityManagerProvider.get().find(KerberosPrincipalHostEntity.class, primaryKey);
- }
-
- /**
- * Find the KerberosPrincipalHostEntity for the requested principal name and host
- *
- * @param principalName a String indicating the name of the requested principal
- * @param hostId a Long indicating the id of the requested host
- * @return the KerberosPrincipalHostEntity or null if not found
- */
- @RequiresSession
- public KerberosPrincipalHostEntity find(String principalName, Long hostId, String keytabPath) {
- return entityManagerProvider.get().find(KerberosPrincipalHostEntity.class,
- new KerberosPrincipalHostEntityPK(principalName, hostId, keytabPath));
- }
-
- /**
- * Find all KerberosPrincipalHostEntities.
- *
- * @return a List of requested KerberosPrincipalHostEntities or null if none were found
- */
- @RequiresSession
- public List<KerberosPrincipalHostEntity> findAll() {
- TypedQuery<KerberosPrincipalHostEntity> query = entityManagerProvider.get().
- createNamedQuery("KerberosPrincipalHostEntityFindAll", KerberosPrincipalHostEntity.class);
-
- return query.getResultList();
- }
-
-
- /**
- * Remove KerberosPrincipalHostEntity instances for the specified principal name
- *
- * @param principalName a String indicating the name of the principal
- */
- @Transactional
- public void removeByPrincipal(String principalName) {
- remove(findByPrincipal(principalName));
- }
-
- /**
- * Remove KerberosPrincipalHostEntity instances for the specified host
- *
- * @param hostId a Long indicating the id of the host
- */
- @Transactional
- public void removeByHost(Long hostId) {
- remove(findByHost(hostId));
- }
-
- /**
- * Remove KerberosPrincipalHostEntity instances for the specified host
- *
- * @param keytabPath a String indicating the keytab path of principal
- */
- @Transactional
- public void removeByKeytabPath(String keytabPath) {
- remove(findByKeytabPath(keytabPath));
- }
- /**
- * Remove KerberosPrincipalHostEntity instance for the specified principal and host
- *
- * @param principalName a String indicating the name of the principal
- * @param hostId a Long indicating the id of the host
- * @see #remove(org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity)
- */
- @Transactional
- public void remove(String principalName, Long hostId, String keytabPath) {
- remove(new KerberosPrincipalHostEntity(principalName, hostId, keytabPath));
- }
-
- /**
- * Tests the existence of a principal on at least one host
- *
- * @param principalName a String indicating the name of the principal to test
- * @return true if a principal is related to one or more hosts; otherwise false
- */
- @RequiresSession
- public boolean exists(String principalName) {
- List<KerberosPrincipalHostEntity> foundEntries = findByPrincipal(principalName);
- return (foundEntries != null) && !foundEntries.isEmpty();
- }
-
- /**
- * Tests the existence of a particular principal on a specific host
- *
- * @param principalName a String indicating the name of the principal to test
- * @param hostId a Long indicating the id of the host to test
- * @return true if the requested principal exists
- */
- @RequiresSession
- public boolean exists(String principalName, Long hostId, String keytabPath) {
- return find(principalName, hostId, keytabPath) != null;
- }
-
- /**
- * Removes multiple KerberosPrincipalHostEntity items
- *
- * @param entities a collection of KerberosPrincipalHostEntity items to remove
- */
- public void remove(List<KerberosPrincipalHostEntity> entities) {
- if (entities != null) {
- for (KerberosPrincipalHostEntity entity : entities) {
- entityManagerProvider.get().remove(entity);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupComponentEntityPK.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupComponentEntityPK.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupComponentEntityPK.java
index 0898133..0d99d79 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupComponentEntityPK.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupComponentEntityPK.java
@@ -18,13 +18,15 @@
package org.apache.ambari.server.orm.entities;
+import java.io.Serializable;
+
import javax.persistence.Column;
import javax.persistence.Id;
/**
* Composite primary key for HostGroupComponentEntity.
*/
-public class HostGroupComponentEntityPK {
+public class HostGroupComponentEntityPK implements Serializable {
@Id
@Column(name = "hostgroup_name", nullable = false, insertable = true, updatable = false, length = 100)
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
index a25931b..1757b9f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.orm.entities;
+import java.util.ArrayList;
import java.util.Collection;
import javax.persistence.CascadeType;
@@ -33,54 +34,113 @@ import javax.persistence.Table;
@Entity
@Table(name = "kerberos_keytab")
@NamedQueries({
- @NamedQuery(name = "KerberosKeytabEntity.findAll", query = "SELECT kk FROM KerberosKeytabEntity kk"),
- @NamedQuery(name = "KerberosKeytabEntity.findByHost",
- query = "SELECT kk FROM KerberosKeytabEntity kk JOIN kk.kerberosPrincipalHostEntities he WHERE he.hostId=:hostId")
+ @NamedQuery(name = "KerberosKeytabEntity.findAll", query = "SELECT kk FROM KerberosKeytabEntity kk"),
+ @NamedQuery(
+ name = "KerberosKeytabEntity.findByPrincipalAndHost",
+ query = "SELECT kk FROM KerberosKeytabEntity kk JOIN kk.kerberosKeytabPrincipalEntities kkp WHERE kkp.hostId=:hostId AND kkp.principalName=:principalName"
+ ),
+ @NamedQuery(
+ name = "KerberosKeytabEntity.findByPrincipalAndNullHost",
+ query = "SELECT kk FROM KerberosKeytabEntity kk JOIN kk.kerberosKeytabPrincipalEntities kkp WHERE kkp.hostId IS NULL AND kkp.principalName=:principalName"
+ )
})
public class KerberosKeytabEntity {
- @Id
- @Column(name = "keytab_path", insertable = true, updatable = false, nullable = false)
- private String keytabPath = null;
-
- @OneToMany(mappedBy = "keytabEntity", cascade = CascadeType.REMOVE, fetch = FetchType.LAZY)
- private Collection<KerberosPrincipalHostEntity> kerberosPrincipalHostEntities;
-
- public KerberosKeytabEntity(){
-
- }
-
- public KerberosKeytabEntity(String keytabPath){
- setKeytabPath(keytabPath);
- }
-
- public String getKeytabPath() {
- return keytabPath;
- }
-
- public void setKeytabPath(String keytabPath) {
- this.keytabPath = keytabPath;
+ @Id
+ @Column(name = "keytab_path", updatable = false, nullable = false)
+ private String keytabPath = null;
+
+ @Column(name = "owner_name")
+ private String ownerName;
+ @Column(name = "owner_access")
+ private String ownerAccess;
+ @Column(name = "group_name")
+ private String groupName;
+ @Column(name = "group_access")
+ private String groupAccess;
+ @Column(name = "is_ambari_keytab")
+ private Integer isAmbariServerKeytab = 0;
+ @Column(name = "write_ambari_jaas")
+ private Integer writeAmbariJaasFile = 0;
+
+ @OneToMany(mappedBy = "kerberosKeytabEntity", cascade = CascadeType.REMOVE, fetch = FetchType.LAZY)
+ private Collection<KerberosKeytabPrincipalEntity> kerberosKeytabPrincipalEntities = new ArrayList<>();
+
+ public KerberosKeytabEntity() {
+
+ }
+
+ public KerberosKeytabEntity(String keytabPath) {
+ setKeytabPath(keytabPath);
+ }
+
+ public String getKeytabPath() {
+ return keytabPath;
+ }
+
+ public void setKeytabPath(String keytabPath) {
+ this.keytabPath = keytabPath;
+ }
+
+ public Collection<KerberosKeytabPrincipalEntity> getKerberosKeytabPrincipalEntities() {
+ return kerberosKeytabPrincipalEntities;
+ }
+
+ public void setKerberosKeytabPrincipalEntities(Collection<KerberosKeytabPrincipalEntity> kerberosKeytabPrincipalEntities) {
+ this.kerberosKeytabPrincipalEntities = kerberosKeytabPrincipalEntities;
+ }
+
+ public String getOwnerName() {
+ return ownerName;
+ }
+
+ public void setOwnerName(String ownerName) {
+ this.ownerName = ownerName;
+ }
+
+ public String getOwnerAccess() {
+ return ownerAccess;
+ }
+
+ public void setOwnerAccess(String ownerAccess) {
+ this.ownerAccess = ownerAccess;
+ }
+
+ public String getGroupName() {
+ return groupName;
+ }
+
+ public void setGroupName(String groupName) {
+ this.groupName = groupName;
+ }
+
+ public String getGroupAccess() {
+ return groupAccess;
+ }
+
+ public void setGroupAccess(String groupAccess) {
+ this.groupAccess = groupAccess;
+ }
+
+ public boolean isAmbariServerKeytab() {
+ return isAmbariServerKeytab == 1;
+ }
+
+ public void setAmbariServerKeytab(boolean ambariServerKeytab) {
+ this.isAmbariServerKeytab = (ambariServerKeytab) ? 1 : 0;
+ }
+
+ public boolean isWriteAmbariJaasFile() {
+ return writeAmbariJaasFile == 1;
+ }
+
+ public void setWriteAmbariJaasFile(boolean writeAmbariJaasFile) {
+ this.writeAmbariJaasFile = (writeAmbariJaasFile) ? 1 : 0;
+ }
+
+ public void addKerberosKeytabPrincipal(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity) {
+ if (!kerberosKeytabPrincipalEntities.contains(kerberosKeytabPrincipalEntity)) {
+ kerberosKeytabPrincipalEntities.add(kerberosKeytabPrincipalEntity);
}
+ }
- public Collection<KerberosPrincipalHostEntity> getKerberosPrincipalHostEntities() {
- return kerberosPrincipalHostEntities;
- }
-
- public void setKerberosPrincipalHostEntities(Collection<KerberosPrincipalHostEntity> kerberosPrincipalHostEntities) {
- this.kerberosPrincipalHostEntities = kerberosPrincipalHostEntities;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- KerberosKeytabEntity that = (KerberosKeytabEntity) o;
-
- return keytabPath.equals(that.keytabPath);
- }
-
- @Override
- public int hashCode() {
- return keytabPath.hashCode();
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabPrincipalEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabPrincipalEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabPrincipalEntity.java
new file mode 100644
index 0000000..9a55587
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabPrincipalEntity.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.orm.entities;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.persistence.CascadeType;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.NamedQueries;
+import javax.persistence.NamedQuery;
+import javax.persistence.OneToMany;
+import javax.persistence.Table;
+import javax.persistence.TableGenerator;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+/**
+ * Represents entity to hold principal for keytab.
+ * Ideally this entity must have natural PK based on ({@link #keytabPath}, {@link #principalName}, {@link #hostId}),
+ * but {@link #hostId} in some cases can be null, and also this entity must be used in service mappings(this can
+ * cause dup of {@link #keytabPath}, {@link #principalName} fields in related entities), so we have surrogate {@link #kkpId}
+ * id and unique constraint on ({@link #keytabPath}, {@link #principalName}, {@link #hostId}).
+ */
+@Entity
+@Table(name = "kerberos_keytab_principal")
+@TableGenerator(name = "kkp_id_generator",
+ table = "ambari_sequences",
+ pkColumnName = "sequence_name",
+ valueColumnName = "sequence_value",
+ pkColumnValue = "kkp_id_seq"
+)
+@NamedQueries({
+ @NamedQuery(
+ name = "KerberosKeytabPrincipalEntity.findAll",
+ query = "SELECT kkpe FROM KerberosKeytabPrincipalEntity kkpe"
+ ),
+ @NamedQuery(
+ name = "KerberosKeytabPrincipalEntity.findByHostAndKeytab",
+ query = "SELECT kkpe FROM KerberosKeytabPrincipalEntity kkpe WHERE kkpe.hostId=:hostId AND kkpe.keytabPath=:keytabPath"
+ ),
+ @NamedQuery(
+ name = "KerberosKeytabPrincipalEntity.findByPrincipal",
+ query = "SELECT kkpe FROM KerberosKeytabPrincipalEntity kkpe WHERE kkpe.principalName=:principalName"
+ ),
+ @NamedQuery(
+ name = "KerberosKeytabPrincipalEntity.findByHost",
+ query = "SELECT kkpe FROM KerberosKeytabPrincipalEntity kkpe WHERE kkpe.hostId=:hostId"
+ ),
+ @NamedQuery(
+ name = "KerberosKeytabPrincipalEntity.findByHostKeytabAndPrincipal",
+ query = "SELECT kkpe FROM KerberosKeytabPrincipalEntity kkpe WHERE kkpe.hostId=:hostId AND kkpe.keytabPath=:keytabPath AND kkpe.principalName=:principalName"
+ ),
+ @NamedQuery(
+ name = "KerberosKeytabPrincipalEntity.findByKeytabAndPrincipalNullHost",
+ query = "SELECT kkpe FROM KerberosKeytabPrincipalEntity kkpe WHERE kkpe.principalName=:principalName AND kkpe.keytabPath=:keytabPath AND kkpe.hostId IS NULL"
+ )
+})
+public class KerberosKeytabPrincipalEntity {
+ @Id
+ @GeneratedValue(strategy = GenerationType.TABLE, generator = "kkp_id_generator")
+ @Column(name = "kkp_id")
+ private Long kkpId;
+
+ @Column(name = "keytab_path", updatable = false, nullable = false)
+ private String keytabPath;
+
+ @Column(name = "principal_name", updatable = false, nullable = false)
+ private String principalName;
+
+ @Column(name = "host_id")
+ private Long hostId;
+
+ @Column(name = "is_distributed", nullable = false)
+ private Integer isDistributed = 0;
+
+ @ManyToOne
+ @JoinColumn(name = "keytab_path", referencedColumnName = "keytab_path", updatable = false, nullable = false, insertable = false)
+ private KerberosKeytabEntity kerberosKeytabEntity;
+
+ @ManyToOne
+ @JoinColumn(name = "host_id", referencedColumnName = "host_id", updatable = false, insertable = false)
+ private HostEntity hostEntity;
+
+ @ManyToOne
+ @JoinColumn(name = "principal_name", referencedColumnName = "principal_name", updatable = false, nullable = false, insertable = false)
+ private KerberosPrincipalEntity principalEntity;
+
+ @OneToMany(cascade = CascadeType.ALL, mappedBy = "kerberosKeytabPrincipalEntity")
+ private List<KerberosKeytabServiceMappingEntity> serviceMapping = new ArrayList<>();
+
+ public KerberosKeytabPrincipalEntity() {
+
+ }
+
+ public KerberosKeytabPrincipalEntity(
+ KerberosKeytabEntity kerberosKeytabEntity,
+ HostEntity hostEntity,
+ KerberosPrincipalEntity principalEntity
+ ) {
+ setKerberosKeytabEntity(kerberosKeytabEntity);
+ setHostEntity(hostEntity);
+ setPrincipalEntity(principalEntity);
+ }
+
+ public Long getKkpId() {
+ return kkpId;
+ }
+
+ public void setKkpId(Long kkpId) {
+ this.kkpId = kkpId;
+ }
+
+ public Boolean isDistributed() {
+ return isDistributed == 1;
+ }
+
+ public void setDistributed(Boolean isDistributed) {
+ this.isDistributed = isDistributed ? 1 : 0;
+ }
+
+ public KerberosKeytabEntity getKerberosKeytabEntity() {
+ return kerberosKeytabEntity;
+ }
+
+ public void setKerberosKeytabEntity(KerberosKeytabEntity kke) {
+ this.kerberosKeytabEntity = kke;
+ if (kke != null) {
+ keytabPath = kke.getKeytabPath();
+ }
+ }
+
+ public HostEntity getHostEntity() {
+ return hostEntity;
+ }
+
+ public void setHostEntity(HostEntity hostEntity) {
+ this.hostEntity = hostEntity;
+ if (hostEntity != null) {
+ hostId = hostEntity.getHostId();
+ }
+ }
+
+ public KerberosPrincipalEntity getPrincipalEntity() {
+ return principalEntity;
+ }
+
+ public void setPrincipalEntity(KerberosPrincipalEntity principalEntity) {
+ this.principalEntity = principalEntity;
+ if (principalEntity != null) {
+ principalName = principalEntity.getPrincipalName();
+ }
+ }
+
+ public String getKeytabPath() {
+ return kerberosKeytabEntity != null ? kerberosKeytabEntity.getKeytabPath() : null;
+ }
+
+
+ public String getPrincipalName() {
+ return principalEntity != null ? principalEntity.getPrincipalName() : null;
+ }
+
+ public Long getHostId() {
+ return hostEntity != null ? hostEntity.getHostId() : null;
+ }
+
+ public String getHostName() {
+ return hostEntity != null ? hostEntity.getHostName() : null;
+ }
+
+ public boolean putServiceMapping(String service, String component) {
+ if (containsMapping(service, component)) {
+ return false;
+ } else {
+ serviceMapping.add(new KerberosKeytabServiceMappingEntity(this, service, component));
+ return true;
+ }
+ }
+
+ public Multimap<String, String> getServiceMappingAsMultimap() {
+ Multimap<String, String> result = ArrayListMultimap.create();
+ for (KerberosKeytabServiceMappingEntity mappingEntity : serviceMapping) {
+ result.put(mappingEntity.getServiceName(), mappingEntity.getComponentName());
+ }
+ return result;
+ }
+
+ public boolean containsMapping(String serviceName, String componentName) {
+ for (KerberosKeytabServiceMappingEntity mappingEntity : serviceMapping) {
+ if (Objects.equal(mappingEntity.getComponentName(), componentName)
+ && Objects.equal(mappingEntity.getServiceName(), serviceName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ KerberosKeytabPrincipalEntity that = (KerberosKeytabPrincipalEntity) o;
+ return Objects.equal(keytabPath, that.keytabPath) &&
+ Objects.equal(principalName, that.principalName) &&
+ Objects.equal(hostId, that.hostId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(keytabPath, principalName, hostId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabServiceMappingEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabServiceMappingEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabServiceMappingEntity.java
new file mode 100644
index 0000000..f3ad7b7
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabServiceMappingEntity.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.entities;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "kkp_mapping_service")
+public class KerberosKeytabServiceMappingEntity {
+ @Id
+ @Column(name = "kkp_id", nullable = false, insertable = false, updatable = false)
+ private Long kerberosKeytabPrincipalId;
+
+ @Id
+ @Column(name = "service_name", nullable = false)
+ private String serviceName;
+
+ @Id
+ @Column(name = "component_name", nullable = false)
+ private String componentName;
+
+ @ManyToOne
+ @JoinColumn(name = "kkp_id")
+ private KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity;
+
+ public KerberosKeytabServiceMappingEntity() {
+ }
+
+ public KerberosKeytabServiceMappingEntity(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity, String serviceName, String componentName) {
+ this.kerberosKeytabPrincipalId = kerberosKeytabPrincipalEntity.getKkpId();
+ this.kerberosKeytabPrincipalEntity = kerberosKeytabPrincipalEntity;
+ this.serviceName = serviceName;
+ this.componentName = componentName;
+ }
+
+ public Long getKerberosKeytabPrincipalId() {
+ return kerberosKeytabPrincipalId;
+ }
+
+ public void setKerberosKeytabPrincipalId(Long kerberosKeytabPrincipalId) {
+ this.kerberosKeytabPrincipalId = kerberosKeytabPrincipalId;
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ public String getComponentName() {
+ return componentName;
+ }
+
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ public KerberosKeytabPrincipalEntity getKerberosKeytabPrincipalEntity() {
+ return kerberosKeytabPrincipalEntity;
+ }
+
+ public void setKerberosKeytabPrincipalEntity(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity) {
+ this.kerberosKeytabPrincipalEntity = kerberosKeytabPrincipalEntity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalEntity.java
index 5dd54ca..5f7cc56 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalEntity.java
@@ -18,16 +18,11 @@
package org.apache.ambari.server.orm.entities;
-import java.util.Collection;
-
-import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
-import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.NamedQueries;
import javax.persistence.NamedQuery;
-import javax.persistence.OneToMany;
import javax.persistence.Table;
/**
@@ -54,9 +49,6 @@ public class KerberosPrincipalEntity {
@Column(name = "cached_keytab_path", insertable = true, updatable = true, nullable = true)
private String cachedKeytabPath = null;
- @OneToMany(mappedBy = "principalEntity", cascade = CascadeType.REMOVE, fetch = FetchType.LAZY)
- private Collection<KerberosPrincipalHostEntity> kerberosPrincipalHostEntities;
-
/**
* Constructs an empty KerberosPrincipalEntity
*/
@@ -130,21 +122,4 @@ public class KerberosPrincipalEntity {
this.cachedKeytabPath = cachedKeytabPath;
}
- /**
- * Gets the list of related KerberosPrincipalHostEntities
- *
- * @return a List of related KerberosPrincipalHostEntities or null if none exist
- */
- public Collection<KerberosPrincipalHostEntity> getKerberosPrincipalHostEntities() {
- return kerberosPrincipalHostEntities;
- }
-
- /**
- * Sets the list of related KerberosPrincipalHostEntities
- *
- * @param kerberosPrincipalHostEntities a List of related KerberosPrincipalHostEntities or null if none exist
- */
- public void setKerberosPrincipalHostEntities(Collection<KerberosPrincipalHostEntity> kerberosPrincipalHostEntities) {
- this.kerberosPrincipalHostEntities = kerberosPrincipalHostEntities;
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
deleted file mode 100644
index d4e80c6..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.orm.entities;
-
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.IdClass;
-import javax.persistence.JoinColumn;
-import javax.persistence.JoinColumns;
-import javax.persistence.ManyToOne;
-import javax.persistence.NamedQueries;
-import javax.persistence.NamedQuery;
-import javax.persistence.Table;
-
-/**
- * Entity representing a KerberosPrincipal stored on a host.
- */
-@Entity
-@IdClass(KerberosPrincipalHostEntityPK.class)
-@Table(name = "kerberos_principal_host")
-@NamedQueries({
- @NamedQuery(name = "KerberosPrincipalHostEntityFindAll",
- query = "SELECT kph FROM KerberosPrincipalHostEntity kph"),
- @NamedQuery(name = "KerberosPrincipalHostEntityFindByPrincipal",
- query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE kph.principalName=:principalName"),
- @NamedQuery(name = "KerberosPrincipalHostEntityFindByHost",
- query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE kph.hostId=:hostId"),
- @NamedQuery(name = "KerberosPrincipalHostEntityFindByKeytabPath",
- query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE kph.keytabPath=:keytabPath"),
-})
-public class KerberosPrincipalHostEntity {
-
- @Id
- @Column(name = "principal_name", insertable = true, updatable = false, nullable = false)
- private String principalName;
-
- @Id
- @Column(name = "host_id", insertable = true, updatable = false, nullable = false)
- private Long hostId;
-
- @Id
- @Column(name = "keytab_path", updatable = false, nullable = false)
- private String keytabPath;
-
- @ManyToOne
- @JoinColumn(name = "principal_name", referencedColumnName = "principal_name", nullable = false, insertable = false, updatable = false)
- private KerberosPrincipalEntity principalEntity;
-
- @ManyToOne
- @JoinColumn(name = "host_id", referencedColumnName = "host_id", nullable = false, insertable = false, updatable = false)
- private HostEntity hostEntity;
-
- @ManyToOne
- @JoinColumns({
- @JoinColumn(name = "keytab_path", referencedColumnName = "keytab_path", nullable = false, insertable = false, updatable = false)
- })
- private KerberosKeytabEntity keytabEntity;
-
- @Column(name = "is_distributed", insertable = true, updatable = true, nullable = false)
- private Integer isDistributed = 0;
- /**
- * Constucts an empty KerberosPrincipalHostEntity
- */
- public KerberosPrincipalHostEntity() {
- }
-
- /**
- * Constructs a new KerberosPrincipalHostEntity
- *
- * @param principalName a String indicating this KerberosPrincipalHostEntity's principal name
- * @param hostId a Long indicating the KerberosPrincipalHostEntity's host id
- */
- public KerberosPrincipalHostEntity(String principalName, Long hostId, String keytabPath) {
- setPrincipalName(principalName);
- setHostId(hostId);
- setKeytabPath(keytabPath);
- }
-
- /**
- * Constructs a new KerberosPrincipalHostEntity
- *
- * @param principalName a String indicating this KerberosPrincipalHostEntity's principal name
- * @param hostId a Long indicating the KerberosPrincipalHostEntity's host id
- */
- public KerberosPrincipalHostEntity(String principalName, Long hostId, String keytabPath, boolean isDistributed) {
- setPrincipalName(principalName);
- setHostId(hostId);
- setKeytabPath(keytabPath);
- setDistributed(isDistributed);
- }
-
- /**
- * Gets the principal name for this KerberosPrincipalHostEntity
- *
- * @return a String indicating this KerberosPrincipalHostEntity's principal name
- */
- public String getPrincipalName() {
- return principalName;
- }
-
- /**
- * Sets the principal name for this KerberosPrincipalHostEntity
- *
- * @param principalName a String indicating this KerberosPrincipalHostEntity's principal name
- */
- public void setPrincipalName(String principalName) {
- this.principalName = principalName;
- }
-
- /**
- * Gets the host name for this KerberosHostHostEntity
- *
- * @return a String indicating this KerberosHostHostEntity's host name
- */
- public String getHostName() {
- return hostEntity != null ? hostEntity.getHostName() : null;
- }
-
- /**
- * Gets the host id for this KerberosHostHostEntity
- *
- * @return a Long indicating this KerberosHostHostEntity's host id
- */
- public Long getHostId() {
- return hostId;
- }
-
- /**
- * Sets the host id for this KerberosHostHostEntity
- *
- * @param hostId a Long indicating this KerberosHostHostEntity's host id
- */
- public void setHostId(Long hostId) {
- this.hostId = hostId;
- }
-
- /**
- * Gets the related HostEntity
- *
- * @return the related HostEntity
- */
- public HostEntity getHostEntity() {
- return hostEntity;
- }
-
- /**
- * Sets the related HostEntity
- *
- * @param hostEntity the related HostEntity
- */
- public void setHostEntity(HostEntity hostEntity) {
- this.hostEntity = hostEntity;
- }
-
- /**
- * Gets the related KerberosPrincipalEntity
- *
- * @return the related KerberosPrincipalEntity
- */
- public KerberosPrincipalEntity getPrincipalEntity() {
- return principalEntity;
- }
-
- /**
- * Sets the related KerberosPrincipalEntity
- *
- * @param principalEntity the related KerberosPrincipalEntity
- */
- public void setPrincipalEntity(KerberosPrincipalEntity principalEntity) {
- this.principalEntity = principalEntity;
- }
-
- public String getKeytabPath() {
- return keytabPath;
- }
-
- public void setKeytabPath(String keytabPath) {
- this.keytabPath = keytabPath;
- }
-
- public KerberosKeytabEntity getKeytabEntity() {
- return keytabEntity;
- }
-
- public void setKeytabEntity(KerberosKeytabEntity keytabEntity) {
- this.keytabEntity = keytabEntity;
- }
-
- public Boolean getDistributed() {
- return isDistributed == 1;
- }
-
- public void setDistributed(Boolean isDistributed) {
- this.isDistributed = (isDistributed) ? 1 : 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
deleted file mode 100644
index 7e57e4a..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.orm.entities;
-
-import java.io.Serializable;
-
-import javax.persistence.Column;
-import javax.persistence.Id;
-
-/**
- * Composite primary key for KerberosPrincipalHostEntity.
- */
-public class KerberosPrincipalHostEntityPK implements Serializable{
-
- @Id
- @Column(name = "principal_name", insertable = false, updatable = false, nullable = false)
- private String principalName = null;
-
- @Id
- @Column(name = "host_id", insertable = false, updatable = false, nullable = false)
- private Long hostId = null;
-
- @Id
- @Column(name = "keytab_path", insertable = false, updatable = false, nullable = false)
- private String keytabPath = null;
-
- public KerberosPrincipalHostEntityPK() {
- }
-
- public KerberosPrincipalHostEntityPK(String principalName, Long hostId, String keytabPath) {
- setPrincipalName(principalName);
- setHostId(hostId);
- setKeytabPath(keytabPath);
- }
-
- /**
- * Get the name of the associated principal.
- *
- * @return principal name
- */
- public String getPrincipalName() {
- return principalName;
- }
-
- /**
- * Set the name of the associated principal.
- *
- * @param principalName principal name
- */
- public void setPrincipalName(String principalName) {
- this.principalName = principalName;
- }
-
- /**
- * Get the host id.
- *
- * @return host id
- */
- public Long getHostId() {
- return hostId;
- }
-
- /**
- * Set the configuration type.
- *
- * @param hostId host id
- */
- public void setHostId(Long hostId) {
- this.hostId = hostId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- KerberosPrincipalHostEntityPK that = (KerberosPrincipalHostEntityPK) o;
-
- return this.principalName.equals(that.principalName) &&
- this.hostId.equals(that.hostId) && this.keytabPath.equals(that.keytabPath);
- }
-
- @Override
- public int hashCode() {
- return 31 * principalName.hashCode() + hostId.hashCode() + keytabPath.hashCode();
- }
-
- public String getKeytabPath() {
- return keytabPath;
- }
-
- public void setKeytabPath(String keytabPath) {
- this.keytabPath = keytabPath;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
index b8affb4..cffd8e1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
@@ -20,7 +20,6 @@ package org.apache.ambari.server.serveraction.kerberos;
import java.io.File;
import java.io.IOException;
-import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -36,6 +35,7 @@ import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.RootComponent;
import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
@@ -47,7 +47,6 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.reflect.TypeToken;
import com.google.inject.Inject;
public abstract class AbstractPrepareKerberosServerAction extends KerberosServerAction {
@@ -66,7 +65,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
private KerberosConfigDataFileWriterFactory kerberosConfigDataFileWriterFactory;
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext) throws AmbariException {
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext) throws AmbariException {
throw new UnsupportedOperationException();
}
@@ -211,7 +210,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
// create database records for keytabs that must be presented on cluster
for (ResolvedKerberosKeytab keytab : resolvedKeytabs.values()) {
- kerberosHelper.processResolvedKeytab(keytab);
+ kerberosHelper.createResolvedKeytab(keytab);
}
} catch (IOException e) {
String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
@@ -235,30 +234,6 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
}
}
- protected Map<String, ? extends Collection<String>> getServiceComponentFilter() {
- String serializedValue = getCommandParameterValue(SERVICE_COMPONENT_FILTER);
-
- if (serializedValue != null) {
- Type type = new TypeToken<Map<String, ? extends Collection<String>>>() {
- }.getType();
- return StageUtils.getGson().fromJson(serializedValue, type);
- } else {
- return null;
- }
- }
-
- protected Collection<String> getIdentityFilter() {
- String serializedValue = getCommandParameterValue(IDENTITY_FILTER);
-
- if (serializedValue != null) {
- Type type = new TypeToken<Collection<String>>() {
- }.getType();
- return StageUtils.getGson().fromJson(serializedValue, type);
- } else {
- return null;
- }
- }
-
private Map<String, Set<String>> gatherPropertiesToIgnore(List<KerberosIdentityDescriptor> identities,
Map<String, Set<String>> propertiesToIgnore) {
Map<String, Map<String, String>> identityConfigurations = kerberosHelper.getIdentityConfigurations(identities);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
index 002076d..1b7d128 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
@@ -34,6 +34,7 @@ import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.SecurityType;
import org.slf4j.Logger;
@@ -58,8 +59,7 @@ public class CleanupServerAction extends KerberosServerAction {
* <p/>
* This method is not used since the {@link #processIdentities(java.util.Map)} is not invoked
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -70,7 +70,7 @@ public class CleanupServerAction extends KerberosServerAction {
* @throws AmbariException if an error occurs while processing the identity record
*/
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
index 3384152..f6fdecd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
@@ -24,13 +24,20 @@ import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.controller.utilities.KerberosChecker;
+import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
-import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
+import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
+import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
+import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
import org.apache.ambari.server.serveraction.ActionLog;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.utils.ShellCommandUtil;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.codec.digest.DigestUtils;
@@ -47,7 +54,7 @@ import com.google.inject.Inject;
* This class mainly relies on the KerberosServerAction to iterate through metadata identifying
* the Kerberos keytab files that need to be created. For each identity in the metadata, this
* implementation's
- * {@link KerberosServerAction#processIdentity(Map, String, KerberosOperationHandler, Map, Map)}
+ * {@link KerberosServerAction#processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, Map)}
* is invoked attempting the creation of the relevant keytab file.
*/
public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction {
@@ -59,10 +66,16 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction
private final static Logger LOG = LoggerFactory.getLogger(ConfigureAmbariIdentitiesServerAction.class);
@Inject
- private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
+ private KerberosKeytabDAO kerberosKeytabDAO;
@Inject
- private KerberosKeytabDAO kerberosKeytabDAO;
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
+ @Inject
+ private KerberosPrincipalDAO kerberosPrincipalDAO;
+
+ @Inject
+ private HostDAO hostDAO;
/**
* Called to execute this action. Upon invocation, calls
@@ -90,8 +103,7 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction
* It is expected that the {@link CreatePrincipalsServerAction}
* (or similar) and {@link CreateKeytabFilesServerAction} has executed before this action.
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -102,45 +114,39 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction
* @throws AmbariException if an error occurs while processing the identity record
*/
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
throws AmbariException {
CommandReport commandReport = null;
- if (identityRecord != null) {
- String message;
+ if (resolvedPrincipal != null) {
String dataDirectory = getDataDirectoryPath();
-
- if (dataDirectory == null) {
- message = "The data directory has not been set. Generated keytab files can not be stored.";
- LOG.error(message);
- commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
- } else {
-
- String hostName = identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME);
- String serviceName = identityRecord.get(KerberosIdentityDataFileReader.SERVICE);
- if (hostName != null && serviceName.equals("AMBARI")) {
- String destKeytabFilePath = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
+ String hostName = resolvedPrincipal.getHostName();
+ for (Map.Entry<String, String> serviceMappingEntry : resolvedPrincipal.getServiceMapping().entries()){
+ String serviceName = serviceMappingEntry.getKey();
+ // distribute ambari keytabs only if host id is null, otherwise they will
+ // be distributed by usual process using ambari-agent.
+ // TODO check if changes needed for multiple principals in one keytab
+ if (resolvedPrincipal.getHostId() == null && hostName != null && serviceName.equals(RootService.AMBARI.name())) {
+ ResolvedKerberosKeytab keytab = resolvedPrincipal.getResolvedKerberosKeytab();
+ String destKeytabFilePath = resolvedPrincipal.getResolvedKerberosKeytab().getFile();
+ hostName = StageUtils.getHostName();
File hostDirectory = new File(dataDirectory, hostName);
- File srcKeytabFile = new File(hostDirectory, DigestUtils.sha1Hex(destKeytabFilePath));
+ File srcKeytabFile = new File(hostDirectory, DigestUtils.sha256Hex(destKeytabFilePath));
if (srcKeytabFile.exists()) {
- String ownerAccess = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS);
- boolean ownerWritable = "w".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
- boolean ownerReadable = "r".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
- String groupAccess = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS);
- boolean groupWritable = "w".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
- boolean groupReadable = "r".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
-
- installAmbariServerIdentity(evaluatedPrincipal, srcKeytabFile.getAbsolutePath(), destKeytabFilePath,
- identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_NAME), ownerReadable, ownerWritable,
- identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_NAME), groupReadable, groupWritable, actionLog);
-
- if ("AMBARI_SERVER_SELF".equals(identityRecord.get(KerberosIdentityDataFileReader.COMPONENT))) {
+ String ownerAccess = keytab.getOwnerAccess();
+ String groupAccess = keytab.getGroupAccess();
+
+ installAmbariServerIdentity(resolvedPrincipal, srcKeytabFile.getAbsolutePath(), destKeytabFilePath,
+ keytab.getOwnerName(), ownerAccess,
+ keytab.getGroupName(), groupAccess, actionLog);
+
+ if (serviceMappingEntry.getValue().contains("AMBARI_SERVER_SELF")) {
// Create/update the JAASFile...
- configureJAAS(evaluatedPrincipal, destKeytabFilePath, actionLog);
+ configureJAAS(resolvedPrincipal.getPrincipal(), destKeytabFilePath, actionLog);
}
}
}
@@ -158,53 +164,56 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction
* @param srcKeytabFilePath the source location of the ambari server keytab file
* @param destKeytabFilePath the destination location of the ambari server keytab file
* @param ownerName the username for the owner of the generated keytab file
- * @param ownerReadable true if the owner should be able to read this file; otherwise false
- * @param ownerWritable true if the owner should be able to write to this file; otherwise false
+ * @param ownerAccess the user file access, "", "r" or "rw"
* @param groupName the name of the group for the generated keytab file
- * @param groupReadable true if the group should be able to read this file; otherwise false
- * @param groupWritable true if the group should be able to write to this file; otherwise false
+ * @param groupAccess the group file access, "", "r" or "rw"
* @param actionLog the logger
* @return true if success; false otherwise
* @throws AmbariException
*/
- public boolean installAmbariServerIdentity(String principal,
+ public boolean installAmbariServerIdentity(ResolvedKerberosPrincipal principal,
String srcKeytabFilePath,
String destKeytabFilePath,
- String ownerName, boolean ownerReadable, boolean ownerWritable,
- String groupName, boolean groupReadable, boolean groupWritable,
+ String ownerName, String ownerAccess,
+ String groupName, String groupAccess,
ActionLog actionLog) throws AmbariException {
try {
// Copy the keytab file into place (creating the parent directory, if necessary...
+ boolean ownerWritable = "w".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
+ boolean ownerReadable = "r".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
+ boolean groupWritable = "w".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
+ boolean groupReadable = "r".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
+
copyFile(srcKeytabFilePath, destKeytabFilePath);
setFileACL(destKeytabFilePath,
ownerName, ownerReadable, ownerWritable,
groupName, groupReadable, groupWritable);
- String ambariServerHostName = StageUtils.getHostName();
Long ambariServerHostID = ambariServerHostID();
- if (ambariServerHostID == null) {
- String message = String.format("Failed to add the kerberos_principal_host record for %s on " +
- "the Ambari server host since the host id for Ambari server host, %s, was not found." +
- " This is not an error if an Ambari agent is not installed on the Ambari server host.",
- principal, ambariServerHostName);
- LOG.warn(message);
- if (actionLog != null) {
- actionLog.writeStdErr(message);
- }
- } else if (!kerberosPrincipalHostDAO.exists(principal, ambariServerHostID, destKeytabFilePath)) {
- if (!kerberosKeytabDAO.exists(destKeytabFilePath)) {
- kerberosKeytabDAO.create(destKeytabFilePath);
- }
- if(!kerberosPrincipalHostDAO.exists(principal, ambariServerHostID, destKeytabFilePath)) {
- kerberosPrincipalHostDAO.create(
- new KerberosPrincipalHostEntity(principal, ambariServerHostID, destKeytabFilePath, true)
- );
- } else {
- KerberosPrincipalHostEntity kphe = kerberosPrincipalHostDAO.find(principal, ambariServerHostID, destKeytabFilePath);
- kphe.setDistributed(true);
- kerberosPrincipalHostDAO.merge(kphe);
- }
+ HostEntity hostEntity = null;
+ if (ambariServerHostID != null) {
+ hostEntity = hostDAO.findById(ambariServerHostID);
+ }
+
+ KerberosKeytabEntity kke = kerberosKeytabDAO.find(destKeytabFilePath);
+ if (!kerberosKeytabDAO.exists(destKeytabFilePath)) {
+ kke = new KerberosKeytabEntity(destKeytabFilePath);
+ kke.setOwnerName(ownerName);
+ kke.setOwnerAccess(ownerAccess);
+ kke.setGroupName(groupName);
+ kke.setGroupAccess(groupAccess);
+ kerberosKeytabDAO.create(kke);
+ }
+
+ for(Map.Entry<String, String> mapping : principal.getServiceMapping().entries()) {
+ String serviceName = mapping.getKey();
+ String componentName = mapping.getValue();
+ KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal.getPrincipal());
+ KerberosKeytabPrincipalEntity entity = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostEntity, principalEntity);
+ entity.setDistributed(true);
+ entity.putServiceMapping(serviceName, componentName);
+ kerberosKeytabPrincipalDAO.merge(entity);
}
if (actionLog != null) {
[11/51] [abbrv] ambari git commit: AMBARI-22647. Rafactor: Package
Log Search and Log Feeder into jars + cleanup start scripts (oleewere)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
new file mode 100755
index 0000000..128617c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
@@ -0,0 +1,274 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JVM="java"
+
+if [ -x $JAVA_HOME/bin/java ]; then
+ JVM=$JAVA_HOME/bin/java
+fi
+
+if [ "$LOGSEARCH_JAVA_MEM" = "" ]; then
+ LOGSEARCH_JAVA_MEM="-Xmx1g"
+fi
+
+readlinkf(){
+ # get real path on mac OSX
+ perl -MCwd -e 'print Cwd::abs_path shift' "$1";
+}
+
+if [ "$(uname -s)" = 'Linux' ]; then
+ SCRIPT_DIR="`dirname "$(readlink -f "$0")"`"
+else
+ SCRIPT_DIR="`dirname "$(readlinkf "$0")"`"
+fi
+
+LOGSEARCH_ROOT_DIR="`dirname \"$SCRIPT_DIR\"`"
+LOGSEARCH_LIBS_DIR="$LOGSEARCH_ROOT_DIR/libs"
+LOGSEARCH_WEBAPP_DIR="$LOGSEARCH_ROOT_DIR/webapp"
+
+if [ "$LOGSEARCH_CONF_DIR" = "" ]; then
+ if [ -d "$LOGSEARCH_ROOT_DIR/conf" ]; then
+ LOGSEARCH_CONF_DIR="$LOGSEARCH_ROOT_DIR/conf"
+ fi
+fi
+
+if [ -f "$LOGSEARCH_CONF_DIR/logsearch-env.sh" ]; then
+ source $LOGSEARCH_CONF_DIR/logsearch-env.sh
+fi
+
+if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then
+ source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE
+fi
+
+if [ -z "$LOGSEARCH_PID_FILE" ]; then
+ LOGSEARCH_PID_DIR=$HOME
+ export LOGSEARCH_PID_FILE=$LOGSEARCH_PID_DIR/logsearch.pid
+fi
+
+if [ -z "$LOG_FILE" ]; then
+ export LOG_FILE="logsearch.log"
+fi
+
+LOGSEARCH_GC_LOGFILE="logsearch_gc.log"
+
+if [ -z "$LOG_PATH" ]; then
+ LOG_FILE="$HOME/$LOG_FILE"
+ LOGSEARCH_GC_LOGFILE="$HOME/$LOGSEARCH_GC_LOGFILE"
+else
+ LOG_PATH_WITHOUT_SLASH=${LOG_PATH%/}
+ LOG_FILE="$LOG_PATH_WITHOUT_SLASH/$LOG_FILE"
+ LOGSEARCH_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGSEARCH_GC_LOGFILE"
+fi
+
+LOGSEARCH_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGSEARCH_GC_LOGFILE"
+
+function print_usage() {
+ cat << EOF
+
+ Usage: [<command>] [<arguments with flags>]
+
+ commands:
+ start Start Log Search
+ stop Stop Log Search
+ status Check Log Search status (pid file)
+ help Print usage
+
+
+ start command arguments:
+ -d, --debug Start java process in debug mode
+ -f, --foreground Start java process in foreground
+
+EOF
+}
+
+function spinner() {
+ local pid=$1
+ local delay=0.5
+ local spinstr='|/-\'
+ while [ "$(ps aux | awk '{print $2}' | grep -w $pid)" ]; do
+ local temp=${spinstr#?}
+ printf " [%c] " "$spinstr"
+ local spinstr=$temp${spinstr%"$temp"}
+ sleep $delay
+ printf "\b\b\b\b\b\b"
+ done
+ printf " \b\b\b\b"
+}
+
+function status() {
+ echo "Checking Log Search status ..."
+ if [ -f "$LOGSEARCH_PID_FILE" ]; then
+ LOGSEARCH_PID=`cat "$LOGSEARCH_PID_FILE"`
+ else
+ echo "Log Search pid not exists. (probably the process is not running)"
+ exit 1
+ fi
+
+ if ps -p $LOGSEARCH_PID > /dev/null
+ then
+ echo "Log Search process is running. (pid: $LOGSEARCH_PID)"
+ exit 0
+ else
+ echo "Log Search process is not running."
+ exit 1
+ fi
+}
+
+function start() {
+ (status);
+ exit_status=$(echo $?)
+ if [ "$exit_status" = "0" ]; then
+ echo "Skipping start process."
+ exit 0
+ fi
+
+ FG="false"
+ LOGSEARCH_DEBUG_SUSPEND=${LOGSEARCH_DEBUG_SUSPEND:-n}
+ LOGSEARCH_DEBUG_PORT=${LOGSEARCH_DEBUG_PORT:-"5005"}
+
+ if [ "$LOGSEARCH_DEBUG" = "true" ]; then
+ LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND "
+ fi
+
+ if [ "$LOGSEARCH_SSL" = "true" ]; then
+ LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGSEARCH_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGSEARCH_KEYSTORE_TYPE -Djavax.net.ssl.trustStore=$LOGSEARCH_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGSEARCH_TRUSTSTORE_TYPE"
+ fi
+
+ if [ "$LOGSEARCH_JMX" = "true" ]; then
+ LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2099"
+ fi
+
+ if [ $# -gt 0 ]; then
+ while true; do
+ case "$1" in
+ -f|--foreground)
+ FG="true"
+ shift
+ ;;
+ -d|--debug)
+ if [ "$LOGSEARCH_DEBUG" != "true" ]; then
+ LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND "
+ fi
+ shift
+ ;;
+ *)
+ if [ "${1:0:2}" == "-D" ]; then
+ # pass thru any opts that begin with -D (java system props)
+ LOGSEARCH_JAVA_OPTS+=("$1")
+ echo "$LOGSEARCH_JAVA_OPTS"
+ shift
+ else
+ if [ "$1" != "" ]; then
+ print_usage
+ exit 1
+ else
+ break
+ fi
+ fi
+ ;;
+ esac
+ done
+ fi
+
+ if [ $FG == "true" ]; then
+ echo "Starting Log Search... (foreground) pid_file=$LOGSEARCH_PID_FILE"
+ echo "Run command $JVM -cp '$LOGSEARCH_CONF_DIR:$LOGSEARCH_WEBAPP_DIR:$LOGSEARCH_LIBS_DIR/*' $LOGSEARCH_GC_OPTS $LOGSEARCH_JAVA_OPTS $LOGSEARCH_JAVA_MEM org.apache.ambari.logsearch.LogSearch"
+ $JVM -cp "$LOGSEARCH_CONF_DIR:$LOGSEARCH_WEBAPP_DIR:$LOGSEARCH_LIBS_DIR/*" $LOGSEARCH_GC_OPTS $LOGSEARCH_JAVA_OPTS $LOGSEARCH_JAVA_MEM org.apache.ambari.logsearch.LogSearch
+ else
+ echo "Starting Log Search... Output file=$LOG_FILE pid_file=$LOGSEARCH_PID_FILE"
+ echo "Run command nohup $JVM -cp '$LOGSEARCH_CONF_DIR:$LOGSEARCH_WEBAPP_DIR:$LOGSEARCH_LIBS_DIR/*' $LOGSEARCH_GC_OPTS $LOGSEARCH_JAVA_OPTS $LOGSEARCH_JAVA_MEM org.apache.ambari.logsearch.LogSearch"
+ nohup $JVM -cp "$LOGSEARCH_CONF_DIR:$LOGSEARCH_WEBAPP_DIR:$LOGSEARCH_LIBS_DIR/*" $LOGSEARCH_GC_OPTS $LOGSEARCH_JAVA_OPTS $LOGSEARCH_JAVA_MEM org.apache.ambari.logsearch.LogSearch > $LOG_FILE 2>&1 &
+ fi
+}
+
+function stop() {
+ LOGSEARCH_STOP_WAIT=3
+ if [ -f "$LOGSEARCH_PID_FILE" ]; then
+ LOGSEARCH_PID=`cat "$LOGSEARCH_PID_FILE"`
+ fi
+
+ if [ "$LOGSEARCH_PID" != "" ]; then
+ echo -e "Sending stop command to Log Search... Checking PID: $LOGSEARCH_PID."
+ kill $LOGSEARCH_PID
+ (loops=0
+ while true
+ do
+ CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGSEARCH_PID | sort -r | tr -d ' '`
+ if [ "$CHECK_PID" != "" ]; then
+ slept=$((loops * 2))
+ if [ $slept -lt $LOGSEARCH_STOP_WAIT ]; then
+ sleep 2
+ loops=$[$loops+1]
+ else
+ exit # subshell!
+ fi
+ else
+ exit # subshell!
+ fi
+ done) &
+ spinner $!
+ rm -f "$LOGSEARCH_PID_FILE"
+ else
+ echo -e "No Log Search process found to stop."
+ exit 0
+ fi
+
+ CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGSEARCH_PID | sort -r | tr -d ' '`
+ if [ "$CHECK_PID" != "" ]; then
+ echo -e "Log Search process $LOGSEARCH_PID is still running; forcefully killing it now."
+ kill -9 $LOGSEARCH_PID
+ echo "Killed process $LOGSEARCH_PID"
+ rm -f "$LOGSEARCH_PID_FILE"
+ sleep 1
+ else
+ echo "Log Search is stopped."
+ fi
+
+ CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGSEARCH_PID | sort -r | tr -d ' '`
+ if [ "$CHECK_PID" != "" ]; then
+ echo "ERROR: Failed to kill Log Search Java process $LOGSEARCH_PID ... script fails."
+ exit 1
+ fi
+}
+
+if [ $# -gt 0 ]; then
+ SCRIPT_CMD="$1"
+ shift
+else
+ print_usage
+ exit 1
+fi
+
+case $SCRIPT_CMD in
+ start)
+ start ${1+"$@"}
+ ;;
+ stop)
+ stop
+ ;;
+ status)
+ status
+ ;;
+ help)
+ print_usage
+ exit 0
+ ;;
+ *)
+ print_usage
+ exit 1
+ ;;
+
+esac
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/scripts/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/run.sh
deleted file mode 100755
index 241b32f..0000000
--- a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/run.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-curr_dir=`pwd`
-cd `dirname $0`; script_dir=`pwd`; cd $curr_dir
-
-if [ ! -z "$LOGSEARCH_INCLUDE" ]; then
- source $LOGSEARCH_INCLUDE
-fi
-
-if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then
- source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE
-fi
-
-JAVA=java
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
-fi
-
-if [ "$LOGSEARCH_JAVA_MEM" = "" ]; then
- LOGSEARCH_JAVA_MEM="-Xmx1g"
-fi
-
-if [ "$LOGFILE" = "" ]; then
- LOGFILE="/var/log/logsearch/logsearch.out"
- touch $LOGFILE 2> /dev/null
- if [ $? -ne 0 ]; then
- LOGFILE=/tmp/${USER}_logsearch.out
- fi
-fi
-
-
-#Temporarily enabling JMX so we can monitor the memory and CPU utilization of the process
-#JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=3098"
-
-LOGSEARCH_DEBUG_SUSPEND=${LOGSEARCH_DEBUG_SUSPEND:-n}
-if [ "$LOGSEARCH_DEBUG" = "true" ] && [ ! -z "$LOGSEARCH_DEBUG_PORT" ]; then
- LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND "
-fi
-
-if [ "$LOGSEARCH_SSL" = "true" ]; then
- LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGSEARCH_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGSEARCH_KEYSTORE_TYPE -Djavax.net.ssl.trustStore=$LOGSEARCH_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGSEARCH_TRUSTSTORE_TYPE"
-fi
-
-if [ "$PID_FILE" = "" ]; then
- LOGSEARCH_PID_DIR=$HOME
- PID_FILE=$LOGSEARCH_PID_DIR/logsearch-search-$USER.pid
-fi
-
-if [ -f ${PID_FILE} ]; then
- PID=`cat ${PID_FILE}`
- if kill -0 $PID 2>/dev/null; then
- echo "logsearch already running (${PID}) killing..."
- kill $PID 2>/dev/null
- sleep 5
- if kill -0 $PID 2>/dev/null; then
- echo "logsearch still running. Will kill process forcefully in another 10 seconds..."
- sleep 10
- kill -9 $PID 2>/dev/null
- sleep 2
- fi
- fi
-
- if kill -0 $PID 2>/dev/null; then
- echo "ERROR: Even after all efforts to stop logsearch, it is still running. pid=$PID. Please manually kill the service and try again."
- exit 1
- fi
-fi
-
-if [ -z "$LOGSEARCH_CONF_DIR" ]; then
- LOGSEARCH_CONF_DIR="/etc/logsearch/conf"
- if [ ! -d $LOGSEARCH_CONF_DIR ]; then
- if [ -d $script_dir/classes ]; then
- LOGSEARCH_CONF_DIR=$script_dir/classes
- fi
- fi
- echo "LOGSEARCH_CONF_DIR not found. Use default: $LOGSEARCH_CONF_DIR"
-fi
-
-LOGSEARCH_GC_LOGFILE=`dirname $LOGFILE`/logsearch_gc.log
-LOGSEARCH_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGSEARCH_GC_LOGFILE"
-
-echo "Starting logsearch. Output file=$LOGFILE pid_file=$PID_FILE"
-#LOGSEARCH_CLI_CLASSPATH=
-#set -x
-nohup $JAVA -cp "$LOGSEARCH_CLI_CLASSPATH:$LOGSEARCH_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGSEARCH_GC_OPTS $LOGSEARCH_JAVA_MEM $LOGSEARCH_JAVA_OPTS $JMX org.apache.ambari.logsearch.LogSearch $* > $LOGFILE 2>&1 &
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/scripts/stop.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/stop.sh b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/stop.sh
deleted file mode 100755
index 3612f08..0000000
--- a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/stop.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-curr_dir=`pwd`
-cd `dirname $0`; script_dir=`pwd`; cd $curr_dir
-
-if [ "$PID_FILE" = "" ]; then
- LOGSEARCH_PID_DIR=$HOME
- PID_FILE=$LOGSEARCH_PID_DIR/logsearch-search-$USER.pid
-fi
-
-if [ -f ${PID_FILE} ]; then
- PID=`cat ${PID_FILE}`
- if kill -0 $PID 2>/dev/null; then
- echo "logsearch running with process id (${PID}). Killing..."
- kill $PID 2>/dev/null
- sleep 5
- if kill -0 $PID 2>/dev/null; then
- echo "logsearch still running. Will kill process forcefully in another 10 seconds..."
- sleep 10
- kill -9 $PID 2>/dev/null
- sleep 2
- fi
- fi
-
- if kill -0 $PID 2>/dev/null; then
- echo "ERROR: Even after all efforts to stop logsearch, it is still running. pid=$PID. Please manually kill the service and try again."
- exit 1
- fi
-fi
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/Dockerfile b/ambari-logsearch/docker/Dockerfile
index e79f61b..8bfcae3 100644
--- a/ambari-logsearch/docker/Dockerfile
+++ b/ambari-logsearch/docker/Dockerfile
@@ -64,11 +64,9 @@ RUN chmod -R 777 /root/test-config
RUN chmod +x /root/start.sh
ENV SOLR_CONFIG_LOCATION /root/config/solr
-ENV LOGSEARCH_CONFIG_LOCATION /root/config/logsearch
-ENV LOGFEEDER_CONFIG_LOCATION /root/config/logfeeder
+ENV LOGSEARCH_CONF_DIR /root/config/logsearch
+ENV LOGFEEDER_CONF_DIR /root/config/logfeeder
ENV SOLR_INCLUDE /root/config/solr/solr-env.sh
-ENV LOGSEARCH_INCLUDE /root/config/logsearch/logsearch-env.sh
-ENV LOGFEEDER_INCLUDE /root/config/logfeeder/logfeeder-env.sh
RUN mkdir -p /var/run/ambari-logsearch-solr /var/log/ambari-logsearch-solr /var/run/ambari-infra-solr-client \
/var/log/ambari-infra-solr-client /root/logsearch_solr_index/data \
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/all.yml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/all.yml b/ambari-logsearch/docker/all.yml
index 369db39..a5afb9a 100644
--- a/ambari-logsearch/docker/all.yml
+++ b/ambari-logsearch/docker/all.yml
@@ -35,6 +35,4 @@ services:
- $MAVEN_REPOSITORY_LOCATION:/root/.m2
- $AMBARI_LOCATION:/root/ambari
- $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs
- - $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config
- - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-logfeeder/target/classes:/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes
- - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-server/target/classes/org:/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/classes/org
\ No newline at end of file
+ - $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/bin/start.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh
index bea06de..f2acf9b 100644
--- a/ambari-logsearch/docker/bin/start.sh
+++ b/ambari-logsearch/docker/bin/start.sh
@@ -49,6 +49,7 @@ function create_logsearch_configs() {
mkdir /root/config/logsearch
cp /root/test-config/logsearch/log4j.xml /root/config/logsearch/
cp /root/test-config/logsearch/logsearch-env.sh /root/config/logsearch/
+ cp $LOGSEARCH_SERVER_PATH/conf/user_pass.json /root/config/logsearch/user_pass.json
if [ $LOGSEARCH_HTTPS_ENABLED == 'true' ]
then
cp /root/test-config/logsearch/logsearch-https.properties /root/config/logsearch/logsearch.properties
@@ -87,7 +88,7 @@ function generate_keys() {
fi
}
-function start_solr() {
+function start_solr_d() {
echo "Starting Solr..."
/root/solr-$SOLR_VERSION/bin/solr start -cloud -s /root/logsearch_solr_index/data -verbose -force
touch /var/log/ambari-logsearch-solr/solr.log
@@ -102,16 +103,25 @@ function start_solr() {
}
function start_logsearch() {
- $LOGSEARCH_SERVER_PATH/run.sh
+ $LOGSEARCH_SERVER_PATH/bin/logsearch.sh start -f
+}
+
+function start_logsearch_d() {
+ $LOGSEARCH_SERVER_PATH/bin/logsearch.sh start
touch /var/log/ambari-logsearch-portal/logsearch-app.log
}
function start_logfeeder() {
- $LOGFEEDER_PATH/run.sh
+ $LOGFEEDER_PATH/bin/logfeeder.sh start -f
+}
+
+
+function start_logfeeder_d() {
+ $LOGFEEDER_PATH/bin/logfeeder.sh start
touch /var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.log
}
-function start_selenium_server() {
+function start_selenium_server_d() {
nohup java -jar /root/selenium-server-standalone.jar > /var/log/selenium-test.log &
}
@@ -141,7 +151,7 @@ function main() {
echo "Start Solr only.."
export COMPONENT_LOG="solr"
generate_keys
- start_solr
+ start_solr_d
log
;;
"logfeeder")
@@ -150,7 +160,6 @@ function main() {
export COMPONENT_LOG="logfeeder"
generate_keys
start_logfeeder
- log
;;
"logsearch")
create_logsearch_configs
@@ -158,15 +167,14 @@ function main() {
export COMPONENT_LOG="logsearch"
generate_keys
start_logsearch
- log
;;
*)
create_configs
generate_keys
- start_selenium_server
- start_solr
- start_logfeeder
- start_logsearch
+ start_selenium_server_d
+ start_solr_d
+ start_logfeeder_d
+ start_logsearch_d
log
;;
esac
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/docker-compose.yml b/ambari-logsearch/docker/docker-compose.yml
index 725af36..e865ec3 100644
--- a/ambari-logsearch/docker/docker-compose.yml
+++ b/ambari-logsearch/docker/docker-compose.yml
@@ -64,7 +64,6 @@ services:
- $AMBARI_LOCATION:/root/ambari
- $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs
- $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config
- - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-server/target/classes/org:/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/classes/org
logfeeder:
image: ambari-logsearch:v1.0
restart: always
@@ -83,7 +82,6 @@ services:
- $AMBARI_LOCATION:/root/ambari
- $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs
- $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config
- - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-logfeeder/target/classes:/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes
networks:
logsearch-network:
driver: bridge
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/logsearch-logfeeder.yml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/logsearch-logfeeder.yml b/ambari-logsearch/docker/logsearch-logfeeder.yml
index 4d48501..75d4391 100644
--- a/ambari-logsearch/docker/logsearch-logfeeder.yml
+++ b/ambari-logsearch/docker/logsearch-logfeeder.yml
@@ -29,7 +29,6 @@ services:
- $AMBARI_LOCATION:/root/ambari
- $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs
- $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config
- - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-logfeeder/target/classes:/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes
networks:
logsearch-network:
driver: bridge
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/logsearch-server.yml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/logsearch-server.yml b/ambari-logsearch/docker/logsearch-server.yml
index 6a0d71e..00c08a2 100644
--- a/ambari-logsearch/docker/logsearch-server.yml
+++ b/ambari-logsearch/docker/logsearch-server.yml
@@ -29,7 +29,6 @@ services:
- $AMBARI_LOCATION:/root/ambari
- $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs
- $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config
- - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-server/target/classes/org:/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/classes/org
networks:
logsearch-network:
driver: bridge
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh b/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh
index 5beb093..a57b1f1 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh
+++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh
@@ -21,10 +21,11 @@ export LOGFEEDER_PATH=/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/t
export LOGFEEDER_CONF_DIR=/root/config/logfeeder
#Logfile e.g. /var/log/logfeeder.log
-export LOGFILE=/var/log/ambari-logsearch-logfeeder/logfeeder.out
+export LOG_PATH=/var/log/ambari-logsearch-logfeeder
+export LOG_FILE=logfeeder.out
#pid file e.g. /var/run/logfeeder.pid
-export PID_FILE=/var/run/ambari-logsearch-logfeeder/logfeeder.pid
+export LOGFEEDER_PID_FILE=/var/run/ambari-logsearch-logfeeder/logfeeder.pid
export JAVA_HOME=/usr/java/default
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
index 1ae57bb..028cb9f 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
+++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
@@ -16,7 +16,7 @@
cluster.name=cl1
logfeeder.checkpoint.folder=/root/checkpoints
logfeeder.metrics.collector.hosts=
-logfeeder.config.dir=/root/test-config/logfeeder/shipper-conf/
+logfeeder.config.dir=/root/config/logfeeder/shipper-conf/
logfeeder.config.files=shipper-conf/global.config.json,\
shipper-conf/output.config.json
logfeeder.log.filter.enable=true
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh b/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh
index 80f9c4c..2a2efd8 100644
--- a/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh
+++ b/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh
@@ -20,9 +20,10 @@ export LOGSEARCH_PATH=/root/ambari/ambari-logsearch/ambari-logsearch-server/targ
export LOGSEARCH_CONF_DIR=/root/config/logsearch
-export LOGFILE=/var/log/ambari-logsearch-portal/logsearch-app.log
+export LOG_PATH=/var/log/ambari-logsearch-portal
+export LOG_FILE=logsearch-app.log
-export PID_FILE=/var/run/ambari-logsearch-portal/logsearch.pid
+export LOGSEARCH_PID_FILE=/var/run/ambari-logsearch-portal/logsearch.pid
export JAVA_HOME=/usr/java/default
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties b/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
index 5e7853d..6b59ac8 100644
--- a/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
+++ b/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
@@ -36,8 +36,8 @@ logsearch.solr.collection.history=history
logsearch.solr.history.config.name=history
logsearch.collection.history.replication.factor=1
-logsearch.solr.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/solr_configsets
-logsearch.solr.audit.logs.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/solr_configsets
+logsearch.solr.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/conf/solr_configsets
+logsearch.solr.audit.logs.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/conf/solr_configsets
# Metrics
logsearch.solr.metrics.collector.hosts=
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
index 5bde17c..22edf76 100644
--- a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
+++ b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
@@ -31,8 +31,8 @@ logsearch.audit.logs.split.interval.mins=15
logsearch.collection.audit.logs.numshards=3
logsearch.collection.audit.logs.replication.factor=2
-logsearch.solr.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/solr_configsets
-logsearch.solr.audit.logs.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/solr_configsets
+logsearch.solr.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/conf/solr_configsets
+logsearch.solr.audit.logs.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-server/target/package/conf/solr_configsets
# History logs
logsearch.solr.collection.history=history
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
index e308479..4afd7b6 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
@@ -160,6 +160,6 @@
<property-file-name>logfeeder-env.sh.j2</property-file-name>
<property-file-type>text</property-file-type>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="true" update="true"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
index a38f961..b43aaa6 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
@@ -29,7 +29,7 @@
</property>
<property>
<name>logfeeder.checkpoint.folder</name>
- <value>/etc/ambari-logsearch-logfeeder/conf/checkpoints</value>
+ <value>/usr/lib/ambari-logsearch-logfeeder/conf/checkpoints</value>
<description>Checkpoint folder for Log Feeder</description>
<display-name>Log Feeder checkpoint dir</display-name>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
index 5c263db..cfc3504 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
@@ -107,7 +107,7 @@
</property>
<property>
<name>logsearch_truststore_location</name>
- <value>/etc/ambari-logsearch-portal/conf/keys/logsearch.jks</value>
+ <value>/usr/lib/ambari-logsearch-portal/conf/keys/logsearch.jks</value>
<display-name>Log Search trust store location</display-name>
<description>Location of the trust store file.</description>
<on-ambari-upgrade add="true"/>
@@ -133,7 +133,7 @@
</property>
<property>
<name>logsearch_keystore_location</name>
- <value>/etc/ambari-logsearch-portal/conf/keys/logsearch.jks</value>
+ <value>/usr/lib/ambari-logsearch-portal/conf/keys/logsearch.jks</value>
<display-name>Log Search key store location</display-name>
<description>Location of the key store file.</description>
<on-ambari-upgrade add="true"/>
@@ -183,6 +183,6 @@
<property-file-name>logsearch-env.sh.j2</property-file-name>
<property-file-type>text</property-file-type>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="true" update="true"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
index fbc29c8..2f6c13c 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py
@@ -23,7 +23,6 @@ from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.check_process_status import check_process_status
from resource_management.libraries.script.script import Script
from setup_logfeeder import setup_logfeeder
-from logsearch_common import kill_process
class LogFeeder(Script):
def install(self, env):
@@ -43,15 +42,15 @@ class LogFeeder(Script):
env.set_params(params)
self.configure(env)
- Execute((format('{logfeeder_dir}/run.sh'),),
- environment={'LOGFEEDER_INCLUDE': format('{logsearch_logfeeder_conf}/logfeeder-env.sh')},
+ Execute((format('{logfeeder_dir}/bin/logfeeder.sh start'),),
sudo=True)
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- kill_process(params.logfeeder_pid_file, getpass.getuser(), params.logfeeder_log_dir)
+ Execute((format('{logfeeder_dir}/bin/logfeeder.sh stop'),),
+ sudo=True)
def status(self, env):
import status_params
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
index 4410712..a276357 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py
@@ -22,7 +22,6 @@ from resource_management.libraries.functions.check_process_status import check_p
from resource_management.libraries.functions.format import format
from resource_management.libraries.script.script import Script
from setup_logsearch import setup_logsearch
-from logsearch_common import kill_process
class LogSearch(Script):
def install(self, env):
@@ -41,8 +40,7 @@ class LogSearch(Script):
env.set_params(params)
self.configure(env)
- Execute(format("{logsearch_dir}/run.sh"),
- environment={'LOGSEARCH_INCLUDE': format('{logsearch_server_conf}/logsearch-env.sh')},
+ Execute(format("{logsearch_dir}/bin/logsearch.sh start"),
user=params.logsearch_user
)
@@ -50,7 +48,9 @@ class LogSearch(Script):
import params
env.set_params(params)
- kill_process(params.logsearch_pid_file, params.logsearch_user, params.logsearch_log_dir)
+ Execute(format("{logsearch_dir}/bin/logsearch.sh stop"),
+ user=params.logsearch_user
+ )
def status(self, env):
import status_params
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_common.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_common.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_common.py
deleted file mode 100644
index c4e0c16..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_common.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.get_user_call_output import get_user_call_output
-from resource_management.libraries.functions.show_logs import show_logs
-
-def kill_process(pid_file, user, log_dir):
- import params
- """
- Kill the process by pid file, then check the process is running or not. If the process is still running after the kill
- command, it will try to kill with -9 option (hard kill)
- """
- pid = get_user_call_output(format("cat {pid_file}"), user=user, is_checked_call=False)[1]
- process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
-
- kill_cmd = format("{sudo} kill {pid}")
- Execute(kill_cmd,
- not_if=format("! ({process_id_exists_command})"))
- wait_time = 5
-
- hard_kill_cmd = format("{sudo} kill -9 {pid}")
- Execute(hard_kill_cmd,
- not_if=format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"),
- ignore_failures = True)
- try:
- Execute(format("! ({process_id_exists_command})"),
- tries=20,
- try_sleep=3,
- )
- except:
- show_logs(log_dir, user)
- raise
-
- File(pid_file,
- action="delete"
- )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 884d32a..4ea2018 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -56,9 +56,9 @@ credential_store_enabled = False
if 'credentialStoreEnabled' in config:
credential_store_enabled = config['credentialStoreEnabled']
-logsearch_server_conf = "/etc/ambari-logsearch-portal/conf"
+logsearch_server_conf = "/usr/lib/ambari-logsearch-portal/conf"
logsearch_server_keys_folder = logsearch_server_conf + "/keys"
-logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf"
+logsearch_logfeeder_conf = "/usr/lib/ambari-logsearch-logfeeder/conf"
logsearch_logfeeder_keys_folder = logsearch_logfeeder_conf + "/keys"
logsearch_config_set_dir = format("{logsearch_server_conf}/solr_configsets")
@@ -170,7 +170,7 @@ zookeeper_quorum = logsearch_solr_zk_quorum
# logsearch-env configs
logsearch_user = config['configurations']['logsearch-env']['logsearch_user']
logsearch_log_dir = config['configurations']['logsearch-env']['logsearch_log_dir']
-logsearch_log = logsearch_log_dir + '/logsearch.out'
+logsearch_log = 'logsearch.out'
logsearch_debug_enabled = str(config['configurations']['logsearch-env']["logsearch_debug_enabled"]).lower()
logsearch_debug_port = config['configurations']['logsearch-env']["logsearch_debug_port"]
logsearch_app_max_memory = config['configurations']['logsearch-env']['logsearch_app_max_memory']
@@ -304,7 +304,7 @@ logfeeder_json_log_maxbackupindex = default('/configurations/logfeeder-log4j/log
# logfeeder-env configs
logfeeder_log_dir = config['configurations']['logfeeder-env']['logfeeder_log_dir']
-logfeeder_log = logfeeder_log_dir + '/logfeeder.out'
+logfeeder_log = 'logfeeder.out'
logfeeder_max_mem = config['configurations']['logfeeder-env']['logfeeder_max_mem']
solr_service_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_service_logs_enable', True)
solr_audit_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_audit_logs_enable', True)
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index 653d604..52b1d76 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -40,7 +40,7 @@ def setup_logfeeder():
recursive_ownership=True
)
- File(params.logfeeder_log,
+ File(format("{logfeeder_log_dir}/{logfeeder_log}"),
mode=0644,
content=''
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 7738cc1..e804b9a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -53,7 +53,7 @@ def setup_logsearch():
owner=params.logsearch_user,
group=params.user_group)
- File(params.logsearch_log,
+ File(format("{logsearch_log_dir}/{logsearch_log}"),
mode=0644,
owner=params.logsearch_user,
group=params.user_group,
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
index f2dd06b..f9f7e40 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
@@ -21,10 +21,11 @@ export LOGFEEDER_PATH={{logfeeder_dir}}
export LOGFEEDER_CONF_DIR={{logsearch_logfeeder_conf}}
#Logfile e.g. /var/log/logfeeder.log
-export LOGFILE={{logfeeder_log}}
+export LOG_PATH={{logfeeder_log_dir}}
+export LOG_FILE={{logfeeder_log}}
#pid file e.g. /var/run/logfeeder.pid
-export PID_FILE={{logfeeder_pid_file}}
+export LOGFEEDER_PID_FILE={{logfeeder_pid_file}}
export JAVA_HOME={{java64_home}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
index 7d9b5de..e823094 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
@@ -21,9 +21,10 @@ export LOGSEARCH_PATH={{logsearch_dir}}
export LOGSEARCH_CONF_DIR={{logsearch_server_conf}}
-export LOGFILE={{logsearch_log_dir}}/logsearch.log
+export LOG_PATH={{logsearch_log_dir}}
+export LOG_FILE={{logsearch_log}}
-export PID_FILE={{logsearch_pid_file}}
+export LOGSEARCH_PID_FILE={{logsearch_pid_file}}
export JAVA_HOME={{java64_home}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
index bf9d79b..4eecfa2 100644
--- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
@@ -47,7 +47,7 @@ major_stack_version = get_major_version(stack_version_formatted)
service_name = config['serviceName']
# logsearch configuration
-logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf"
+logsearch_logfeeder_conf = "/usr/lib/ambari-logsearch-logfeeder/conf"
agent_cache_dir = config['hostLevelParams']['agentCacheDir']
service_package_folder = config['commandParams']['service_package_folder']
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index 19c81a8..1f1d3f7 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -58,7 +58,7 @@ class TestHookAfterInstall(RMFTestCase):
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /etc/hadoop/conf")
self.assertResourceCalled('Directory',
- '/etc/ambari-logsearch-logfeeder/conf',
+ '/usr/lib/ambari-logsearch-logfeeder/conf',
mode = 0755,
cd_access = 'a',
create_parents = True)
@@ -106,7 +106,7 @@ class TestHookAfterInstall(RMFTestCase):
only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf")
self.assertResourceCalled('Directory',
- '/etc/ambari-logsearch-logfeeder/conf',
+ '/usr/lib/ambari-logsearch-logfeeder/conf',
mode = 0755,
cd_access = 'a',
create_parents = True)
@@ -174,7 +174,7 @@ class TestHookAfterInstall(RMFTestCase):
only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf")
self.assertResourceCalled('Directory',
- '/etc/ambari-logsearch-logfeeder/conf',
+ '/usr/lib/ambari-logsearch-logfeeder/conf',
mode = 0755,
cd_access = 'a',
create_parents = True)
@@ -277,7 +277,7 @@ class TestHookAfterInstall(RMFTestCase):
only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf")
self.assertResourceCalled('Directory',
- '/etc/ambari-logsearch-logfeeder/conf',
+ '/usr/lib/ambari-logsearch-logfeeder/conf',
mode = 0755,
cd_access = 'a',
create_parents = True)
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 159cb81..13774f7 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -37,7 +37,7 @@ class TestLogFeeder(RMFTestCase):
cd_access='a',
mode=0755
)
- self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf/checkpoints',
create_parents=True,
cd_access='a',
mode=0755
@@ -49,7 +49,7 @@ class TestLogFeeder(RMFTestCase):
cd_access='a',
mode=0755
)
- self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf',
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf',
create_parents=True,
recursive_ownership=True,
cd_access='a',
@@ -60,18 +60,18 @@ class TestLogFeeder(RMFTestCase):
mode=0644,
content=''
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/keys/ks_pass.txt',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/keys/ks_pass.txt',
action = ['delete']
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/keys/ts_pass.txt',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/keys/ts_pass.txt',
action = ['delete']
)
- self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
+ self.assertResourceCalled('PropertiesFile', '/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder.properties',
properties={'cluster.name': 'c1',
'common-property': 'common-value',
- 'hadoop.security.credential.provider.path': 'jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder.jceks',
- 'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
- 'logfeeder.config.dir': '/etc/ambari-logsearch-logfeeder/conf',
+ 'hadoop.security.credential.provider.path': 'jceks://file/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder.jceks',
+ 'logfeeder.checkpoint.folder': '/usr/lib/ambari-logsearch-logfeeder/conf/checkpoints',
+ 'logfeeder.config.dir': '/usr/lib/ambari-logsearch-logfeeder/conf',
'logfeeder.config.files': 'output.config.json,global.config.json',
'logfeeder.metrics.collector.hosts': '',
'logfeeder.metrics.collector.path': '/ws/v1/timeline/metrics',
@@ -82,25 +82,25 @@ class TestLogFeeder(RMFTestCase):
'logsearch.config.zk_connect_string': 'c6401.ambari.apache.org:2181'
}
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder-env.sh',
mode=0755,
content=InlineTemplate(self.getConfig()['configurations']['logfeeder-env']['content'])
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/log4j.xml',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/log4j.xml',
content=InlineTemplate(self.getConfig()['configurations']['logfeeder-log4j']['content'])
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/grok-patterns',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/grok-patterns',
content=InlineTemplate('GP'),
encoding='utf-8'
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/global.config.json',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/global.config.json',
content=Template('global.config.json.j2')
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-ambari.json',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-ambari.json',
content=InlineTemplate('ambari-grok-filter'),
encoding='utf-8'
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/output.config.json',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/output.config.json',
content=InlineTemplate('output-grok-filter'),
encoding='utf-8'
)
@@ -127,8 +127,6 @@ class TestLogFeeder(RMFTestCase):
)
self.configureResourcesCalled()
- self.assertResourceCalled('Execute', ('/usr/lib/ambari-logsearch-logfeeder/run.sh',),
- sudo=True,
- environment={
- 'LOGFEEDER_INCLUDE': '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh'}
+ self.assertResourceCalled('Execute', ('/usr/lib/ambari-logsearch-logfeeder/bin/logfeeder.sh start',),
+ sudo=True
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index db19e0f..292773b 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -47,7 +47,7 @@ class TestLogSearch(RMFTestCase):
cd_access = 'a',
mode = 0755
)
- self.assertResourceCalled('Directory', '/etc/ambari-logsearch-portal/conf',
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-portal/conf',
owner = 'logsearch',
group = 'hadoop',
create_parents = True,
@@ -55,7 +55,7 @@ class TestLogSearch(RMFTestCase):
cd_access = 'a',
mode = 0755
)
- self.assertResourceCalled('Directory', '/etc/ambari-logsearch-portal/conf/solr_configsets',
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-portal/conf/solr_configsets',
owner = 'logsearch',
group = 'hadoop',
create_parents = True,
@@ -63,7 +63,7 @@ class TestLogSearch(RMFTestCase):
cd_access = 'a',
mode = 0755
)
- self.assertResourceCalled('Directory', '/etc/ambari-logsearch-portal/conf/keys',
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-portal/conf/keys',
owner = 'logsearch',
group = 'hadoop',
cd_access = 'a',
@@ -76,15 +76,15 @@ class TestLogSearch(RMFTestCase):
mode = 0644,
content = ''
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/keys/ks_pass.txt',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/keys/ks_pass.txt',
action = ['delete']
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/keys/ts_pass.txt',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/keys/ts_pass.txt',
action = ['delete']
)
- self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
+ self.assertResourceCalled('PropertiesFile', '/usr/lib/ambari-logsearch-portal/conf/logsearch.properties',
properties = {'common-property': 'common-value',
- 'hadoop.security.credential.provider.path': 'jceks://file/etc/ambari-logsearch-portal/conf/logsearch.jceks',
+ 'hadoop.security.credential.provider.path': 'jceks://file/usr/lib/ambari-logsearch-portal/conf/logsearch.jceks',
'logsearch.audit.logs.split.interval.mins': '1',
'logsearch.auth.external_auth.enabled': 'false',
'logsearch.auth.external_auth.host_url': 'http://c6401.ambari.apache.org:8080',
@@ -115,38 +115,38 @@ class TestLogSearch(RMFTestCase):
'logsearch.spnego.kerberos.host': 'localhost'
}
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/HadoopServiceConfig.json',
owner = 'logsearch',
group='hadoop',
content = Template('HadoopServiceConfig.json.j2')
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/log4j.xml',
owner = 'logsearch',
group='hadoop',
content = InlineTemplate(self.getConfig()['configurations']['logsearch-log4j']['content'])
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch-env.sh',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/logsearch-env.sh',
content = InlineTemplate(self.getConfig()['configurations']['logsearch-env']['content']),
mode = 0755,
owner = "logsearch",
group='hadoop'
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch-admin.json',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/logsearch-admin.json',
owner = 'logsearch',
group='hadoop',
content = InlineTemplate(self.getConfig()['configurations']['logsearch-admin-json']['content'])
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/solrconfig.xml',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/solrconfig.xml',
owner = 'logsearch',
group='hadoop',
content = InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content'])
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/solrconfig.xml',
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/solrconfig.xml',
owner = 'logsearch',
group='hadoop',
content = InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content'])
)
- self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'),
+ self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/usr/lib/ambari-logsearch-portal/conf/solr_configsets'),
sudo = True
)
self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 30 --interval 5')
@@ -174,7 +174,6 @@ class TestLogSearch(RMFTestCase):
)
self.configureResourcesCalled()
- self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-portal/run.sh",
- environment = {'LOGSEARCH_INCLUDE': '/etc/ambari-logsearch-portal/conf/logsearch-env.sh'},
+ self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-portal/bin/logsearch.sh start",
user = "logsearch"
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-server/src/test/python/stacks/2.4/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json
index 9966f3e..701132a 100644
--- a/ambari-server/src/test/python/stacks/2.4/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json
@@ -315,7 +315,7 @@
"logsearch.roles.allowed": "AMBARI.ADMINISTRATOR,CLUSTER.ADMINISTRATOR"
},
"logfeeder-properties": {
- "logfeeder.checkpoint.folder" : "/etc/ambari-logsearch-logfeeder/conf/checkpoints",
+ "logfeeder.checkpoint.folder" : "/usr/lib/ambari-logsearch-logfeeder/conf/checkpoints",
"logfeeder.metrics.collector.hosts" : "{metrics_collector_hosts}",
"logfeeder.config.files" : "{default_config_files}"
},
[07/51] [abbrv] ambari git commit: AMBARI-22657 Oozie service check
failed during 4th digit PU (dgrinenko)
Posted by am...@apache.org.
AMBARI-22657 Oozie service check failed during 4th digit PU (dgrinenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3ab1045c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3ab1045c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3ab1045c
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 3ab1045c1f95a90159e69fbd934953927947c754
Parents: f1f7302
Author: Dmytro Grinenko <ha...@apache.org>
Authored: Fri Dec 15 17:50:01 2017 +0200
Committer: Dmytro Grinenko <ha...@apache.org>
Committed: Fri Dec 15 17:50:01 2017 +0200
----------------------------------------------------------------------
.../resources/stacks/HDP/2.0.6/properties/stack_packages.json | 3 ++-
.../main/resources/stacks/HDP/3.0/properties/stack_packages.json | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ab1045c/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
index 62a46b9..01a30f3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
@@ -536,7 +536,8 @@
"oozie-server"
],
"PATCH": [
- "oozie-server"
+ "oozie-server",
+ "oozie-client"
],
"STANDARD": [
"oozie-client",
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ab1045c/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
index 4f05377..28efbfa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
@@ -532,7 +532,8 @@
"oozie-server"
],
"PATCH": [
- "oozie-server"
+ "oozie-server",
+ "oozie-client"
],
"STANDARD": [
"oozie-client",
[38/51] [abbrv] ambari git commit: AMBARI-22686. Disabled stack still
appears in the UI if VDF is available. (swagle)
Posted by am...@apache.org.
AMBARI-22686. Disabled stack still appears in the UI if VDF is available. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e404100b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e404100b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e404100b
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: e404100b99ffac8f14e36b46770466deeb5760d7
Parents: 36d0271
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Dec 21 13:30:14 2017 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Dec 21 13:30:14 2017 -0800
----------------------------------------------------------------------
.../server/api/services/AmbariMetaInfo.java | 12 ++++----
.../server/api/services/AmbariMetaInfoTest.java | 32 ++++++++++++++++++--
.../RepositoryVersionResourceProviderTest.java | 5 +--
3 files changed, 37 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e404100b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index b1eba8f..9fee0ae 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -139,7 +139,7 @@ public class AmbariMetaInfo {
private File commonWidgetsDescriptorFile;
private File customActionRoot;
private String commonKerberosDescriptorFileLocation;
- private Map<String, VersionDefinitionXml> versionDefinitions = null;
+ Map<String, VersionDefinitionXml> versionDefinitions = null;
@Inject
@@ -1368,12 +1368,12 @@ public class AmbariMetaInfo {
versionDefinitions = new HashMap<>();
for (StackInfo stack : getStacks()) {
- for (VersionDefinitionXml definition : stack.getVersionDefinitions()) {
- versionDefinitions.put(String.format("%s-%s-%s", stack.getName(),
- stack.getVersion(), definition.release.version), definition);
- }
-
if (stack.isActive() && stack.isValid()) {
+ for (VersionDefinitionXml definition : stack.getVersionDefinitions()) {
+ versionDefinitions.put(String.format("%s-%s-%s", stack.getName(),
+ stack.getVersion(), definition.release.version), definition);
+ }
+
try {
// !!! check for a "latest-vdf" one. This will be used for the default if one is not found.
VersionDefinitionXml xml = stack.getLatestVersionDefinition();
http://git-wip-us.apache.org/repos/asf/ambari/blob/e404100b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
index 9285526..f98cffd 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
@@ -133,8 +133,7 @@ public class AmbariMetaInfoTest {
private static final int OS_CNT = 4;
private static TestAmbariMetaInfo metaInfo = null;
- private final static Logger LOG =
- LoggerFactory.getLogger(AmbariMetaInfoTest.class);
+ private final static Logger LOG = LoggerFactory.getLogger(AmbariMetaInfoTest.class);
private static final String FILE_NAME = "hbase-site.xml";
private static final String HADOOP_ENV_FILE_NAME = "hadoop-env.xml";
private static final String HDFS_LOG4J_FILE_NAME = "hdfs-log4j.xml";
@@ -1902,6 +1901,35 @@ public class AmbariMetaInfoTest {
Assert.assertEquals("src/test/resources/widgets.json", widgetsFile.getPath());
}
+ @Test
+ public void testGetVersionDefinitionsForDisabledStack() throws AmbariException {
+ Map<String, VersionDefinitionXml> versionDefinitions = metaInfo.getVersionDefinitions();
+ Assert.assertNotNull(versionDefinitions);
+ // Check presence
+ Map.Entry<String, VersionDefinitionXml> vdfEntry = null;
+ for (Map.Entry<String, VersionDefinitionXml> entry : versionDefinitions.entrySet()) {
+ if (entry.getKey().equals("HDP-2.2.1")) {
+ vdfEntry = entry;
+ }
+ }
+ Assert.assertNotNull("Candidate stack and vdf for test case.", vdfEntry);
+ StackInfo stackInfo = metaInfo.getStack("HDP", "2.2.1");
+ // Strange that this is not immutable but works for this test !
+ stackInfo.setActive(false);
+
+ // Hate to use reflection hence changed contract to be package private
+ metaInfo.versionDefinitions = null;
+
+ versionDefinitions = metaInfo.getVersionDefinitions();
+ vdfEntry = null;
+ for (Map.Entry<String, VersionDefinitionXml> entry : versionDefinitions.entrySet()) {
+ if (entry.getKey().equals("HDP-2.2.1")) {
+ vdfEntry = entry;
+ }
+ }
+ Assert.assertNull("Disabled stack should not be returned by the API", vdfEntry);
+ }
+
private File getStackRootTmp(String buildDir) {
return new File(buildDir + "/ambari-metaInfo");
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e404100b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java
index 6bc8b95..243b060 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java
@@ -229,9 +229,6 @@ public class RepositoryVersionResourceProviderTest {
Assert.assertEquals(1, provider.getResources(getRequest, new AndPredicate(predicateStackName, predicateStackVersion)).size());
}
-
-
-
@Test
public void testGetResourcesAsAdministrator() throws Exception {
testGetResources(TestAuthenticationFactory.createAdministrator());
@@ -357,7 +354,7 @@ public class RepositoryVersionResourceProviderTest {
RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity3);
}
-
+
@Test
public void testDeleteResourcesAsAdministrator() throws Exception {
testDeleteResources(TestAuthenticationFactory.createAdministrator());
[20/51] [abbrv] ambari git commit: Updating Ambari docs for Ambari
2.6.1 and Ambari 2.6.0
Posted by am...@apache.org.
Updating Ambari docs for Ambari 2.6.1 and Ambari 2.6.0
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/90554f38
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/90554f38
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/90554f38
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 90554f3810c140787cf4550358952c4438ca78e0
Parents: be0e87b
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Mon Dec 18 22:39:17 2017 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Mon Dec 18 22:40:15 2017 -0800
----------------------------------------------------------------------
docs/pom.xml | 2 +-
docs/src/site/apt/index.apt | 2 +-
docs/src/site/apt/whats-new.apt | 14 +++++++++++++-
docs/src/site/site.xml | 4 ++++
4 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/90554f38/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index f3de652..e24a340 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -28,7 +28,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.ambari</groupId>
- <version>2.5.2</version>
+ <version>2.6.1</version>
<artifactId>ambari</artifactId>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/ambari/blob/90554f38/docs/src/site/apt/index.apt
----------------------------------------------------------------------
diff --git a/docs/src/site/apt/index.apt b/docs/src/site/apt/index.apt
index d9bbe6d..3ffe78e 100644
--- a/docs/src/site/apt/index.apt
+++ b/docs/src/site/apt/index.apt
@@ -52,7 +52,7 @@ Introduction
Getting Started with Ambari
- Follow the {{{https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.5.2} installation guide for Ambari 2.5.2}}.
+ Follow the {{{https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.6.1} installation guide for Ambari 2.6.1}}.
Note: Ambari currently supports the 64-bit version of the following Operating Systems:
http://git-wip-us.apache.org/repos/asf/ambari/blob/90554f38/docs/src/site/apt/whats-new.apt
----------------------------------------------------------------------
diff --git a/docs/src/site/apt/whats-new.apt b/docs/src/site/apt/whats-new.apt
index a1625ce..6ab75a1 100644
--- a/docs/src/site/apt/whats-new.apt
+++ b/docs/src/site/apt/whats-new.apt
@@ -16,7 +16,19 @@
What's New with Ambari?
- * The latest release of Ambari is Ambari 2.5.2.
+ * The latest release of Ambari is Ambari 2.6.1.
+
+ * Ambari 2.6.1 added the following feature:
+
+ * Conditionally Install LZO packages through Ambari (AMBARI-22457)
+
+ * Ambari 2.6.0 added the following features:
+
+ * Ambari Metrics System's (AMS) distributed mode with multiple Collectors (General Availability) (AMBARI-15901)
+
+ * Improvements in Host Recovery (Restart only), moving masters with minimum impact and scale testing (AMBARI-21898, AMBARI-21929, AMBARI-21834)
+
+ * Improvement in Data Archival & Purging in Ambari Infra (AMBARI-21810)
* Ambari 2.5.2 added the following features:
http://git-wip-us.apache.org/repos/asf/ambari/blob/90554f38/docs/src/site/site.xml
----------------------------------------------------------------------
diff --git a/docs/src/site/site.xml b/docs/src/site/site.xml
index f2d0d07..66d7ad4 100644
--- a/docs/src/site/site.xml
+++ b/docs/src/site/site.xml
@@ -107,6 +107,8 @@
<item name="Quick Start Guide" href="https://cwiki.apache.org/confluence/display/AMBARI/Quick+Start+Guide"/>
<item name="Features + Roadmap" href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=30755705"/>
<item name="API Reference" href="https://github.com/apache/ambari/blob/trunk/ambari-server/docs/api/v1/index.md"/>
+ <item name="Install Guide for 2.6.1" href="https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.6.1"/>
+ <item name="Install Guide for 2.6.0" href="https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.6.0"/>
<item name="Install Guide for 2.5.2" href="https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.5.2"/>
<item name="Install Guide for 2.5.1" href="https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.5.1"/>
<item name="Install Guide for 2.5.0" href="https://cwiki.apache.org/confluence/display/AMBARI/Installation+Guide+for+Ambari+2.5.0"/>
@@ -142,6 +144,8 @@
</menu>
<menu name="Releases">
+ <item name="2.6.1" href="http://www.apache.org/dyn/closer.cgi/ambari/ambari-2.6.1"/>
+ <item name="2.6.0" href="http://www.apache.org/dyn/closer.cgi/ambari/ambari-2.6.0"/>
<item name="2.5.2" href="http://www.apache.org/dyn/closer.cgi/ambari/ambari-2.5.2"/>
<item name="2.5.1" href="http://www.apache.org/dyn/closer.cgi/ambari/ambari-2.5.1"/>
<item name="2.5.0" href="http://www.apache.org/dyn/closer.cgi/ambari/ambari-2.5.0"/>
[32/51] [abbrv] ambari git commit: AMBARI-22530. Refactor internal
code of handling info between kerberos wizard actions (echekanskiy)
Posted by am...@apache.org.
AMBARI-22530. Refactor internal code of handling info between kerberos wizard actions (echekanskiy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/67fc4a37
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/67fc4a37
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/67fc4a37
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 67fc4a3785da0a7c39dcb27f220c8573a59ab63d
Parents: 81c0454
Author: root <ro...@build.home.lan>
Authored: Thu Dec 21 10:58:23 2017 -0500
Committer: Eugene Chekanskiy <ec...@hortonworks.com>
Committed: Thu Dec 21 11:00:37 2017 -0500
----------------------------------------------------------------------
.../ambari/server/agent/HeartBeatHandler.java | 122 +-
.../ambari/server/agent/HeartbeatProcessor.java | 33 +-
.../controller/DeleteIdentityHandler.java | 5 +-
.../server/controller/KerberosHelper.java | 2 +-
.../server/controller/KerberosHelperImpl.java | 1129 +++++++++---------
.../HostKerberosIdentityResourceProvider.java | 15 +-
.../server/orm/dao/KerberosKeytabDAO.java | 154 ++-
.../orm/dao/KerberosKeytabPrincipalDAO.java | 309 +++++
.../server/orm/dao/KerberosPrincipalDAO.java | 9 -
.../orm/dao/KerberosPrincipalHostDAO.java | 252 ----
.../entities/HostGroupComponentEntityPK.java | 4 +-
.../orm/entities/KerberosKeytabEntity.java | 152 ++-
.../entities/KerberosKeytabPrincipalEntity.java | 236 ++++
.../KerberosKeytabServiceMappingEntity.java | 88 ++
.../orm/entities/KerberosPrincipalEntity.java | 25 -
.../entities/KerberosPrincipalHostEntity.java | 213 ----
.../entities/KerberosPrincipalHostEntityPK.java | 115 --
.../AbstractPrepareKerberosServerAction.java | 31 +-
.../kerberos/CleanupServerAction.java | 6 +-
.../ConfigureAmbariIdentitiesServerAction.java | 141 ++-
.../kerberos/CreateKeytabFilesServerAction.java | 112 +-
.../kerberos/CreatePrincipalsServerAction.java | 47 +-
.../kerberos/DestroyPrincipalsServerAction.java | 62 +-
.../kerberos/FinalizeKerberosServerAction.java | 24 +-
.../kerberos/KerberosServerAction.java | 291 ++---
.../PrepareEnableKerberosServerAction.java | 16 +-
.../PrepareKerberosIdentitiesServerAction.java | 9 -
.../stageutils/KerberosKeytabController.java | 213 ++++
.../stageutils/ResolvedKerberosKeytab.java | 117 +-
.../stageutils/ResolvedKerberosPrincipal.java | 169 +++
.../upgrades/PreconfigureKerberosAction.java | 12 +-
.../server/state/cluster/ClustersImpl.java | 8 +-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 34 +-
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 33 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 35 +-
.../resources/Ambari-DDL-Postgres-CREATE.sql | 35 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 33 +-
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 33 +-
.../src/main/resources/META-INF/persistence.xml | 3 +-
.../server/agent/TestHeartbeatHandler.java | 79 +-
.../server/controller/KerberosHelperTest.java | 47 +-
...ostKerberosIdentityResourceProviderTest.java | 15 +-
.../apache/ambari/server/orm/db/DDLTests.java | 2 +-
...nfigureAmbariIdentitiesServerActionTest.java | 36 +-
.../FinalizeKerberosServerActionTest.java | 5 +-
.../kerberos/KerberosServerActionTest.java | 26 +-
.../PreconfigureKerberosActionTest.java | 16 +-
47 files changed, 2618 insertions(+), 1935 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
index 53cceb0..2b82fe3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
@@ -26,6 +26,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,8 +40,10 @@ import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileReader;
-import org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileReaderFactory;
import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.AgentVersion;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
@@ -104,11 +107,8 @@ public class HeartBeatHandler {
@Inject
private RecoveryConfigHelper recoveryConfigHelper;
- /**
- * KerberosIdentityDataFileReaderFactory used to create KerberosIdentityDataFileReader instances
- */
@Inject
- private KerberosIdentityDataFileReaderFactory kerberosIdentityDataFileReaderFactory;
+ private KerberosKeytabController kerberosKeytabController;
private Map<String, Long> hostResponseIds = new ConcurrentHashMap<>();
@@ -241,7 +241,6 @@ public class HeartBeatHandler {
* TODO: Handle the case when a host is a part of multiple clusters.
*/
Set<Cluster> clusters = clusterFsm.getClustersForHost(hostname);
-
if (clusters.size() > 0) {
String clusterName = clusters.iterator().next().getClusterName();
@@ -584,80 +583,75 @@ public class HeartBeatHandler {
*/
void injectKeytab(ExecutionCommand ec, String command, String targetHost) throws AmbariException {
String dataDir = ec.getCommandParams().get(KerberosServerAction.DATA_DIRECTORY);
-
+ KerberosServerAction.KerberosCommandParameters kerberosCommandParameters = new KerberosServerAction.KerberosCommandParameters(ec);
if(dataDir != null) {
- KerberosIdentityDataFileReader reader = null;
List<Map<String, String>> kcp = ec.getKerberosCommandParams();
try {
- reader = kerberosIdentityDataFileReaderFactory.createKerberosIdentityDataFileReader(new File(dataDir, KerberosIdentityDataFileReader.DATA_FILE_NAME));
-
- for (Map<String, String> record : reader) {
- String hostName = record.get(KerberosIdentityDataFileReader.HOSTNAME);
-
- if (targetHost.equalsIgnoreCase(hostName)) {
-
- if (SET_KEYTAB.equalsIgnoreCase(command)) {
- String keytabFilePath = record.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
-
- if (keytabFilePath != null) {
-
- String sha1Keytab = DigestUtils.sha1Hex(keytabFilePath);
- File keytabFile = new File(dataDir + File.separator + hostName + File.separator + sha1Keytab);
-
- if (keytabFile.canRead()) {
- Map<String, String> keytabMap = new HashMap<>();
- String principal = record.get(KerberosIdentityDataFileReader.PRINCIPAL);
- String isService = record.get(KerberosIdentityDataFileReader.SERVICE);
-
+ Set<ResolvedKerberosKeytab> keytabsToInject = kerberosKeytabController.getFilteredKeytabs((Map<String, Collection<String>>)kerberosCommandParameters.getServiceComponentFilter(), kerberosCommandParameters.getHostFilter(), kerberosCommandParameters.getIdentityFilter());
+ for (ResolvedKerberosKeytab resolvedKeytab : keytabsToInject) {
+ for(ResolvedKerberosPrincipal resolvedPrincipal: resolvedKeytab.getPrincipals()) {
+ String hostName = resolvedPrincipal.getHostName();
+
+ if (targetHost.equalsIgnoreCase(hostName)) {
+
+ if (SET_KEYTAB.equalsIgnoreCase(command)) {
+ String keytabFilePath = resolvedKeytab.getFile();
+
+ if (keytabFilePath != null) {
+
+ String sha1Keytab = DigestUtils.sha256Hex(keytabFilePath);
+ File keytabFile = new File(dataDir + File.separator + hostName + File.separator + sha1Keytab);
+
+ if (keytabFile.canRead()) {
+ Map<String, String> keytabMap = new HashMap<>();
+ String principal = resolvedPrincipal.getPrincipal();
+
+ keytabMap.put(KerberosIdentityDataFileReader.HOSTNAME, hostName);
+ keytabMap.put(KerberosIdentityDataFileReader.PRINCIPAL, principal);
+ keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH, keytabFilePath);
+ keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_NAME, resolvedKeytab.getOwnerName());
+ keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS, resolvedKeytab.getOwnerAccess());
+ keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_NAME, resolvedKeytab.getGroupName());
+ keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_ACCESS, resolvedKeytab.getGroupAccess());
+
+ BufferedInputStream bufferedIn = new BufferedInputStream(new FileInputStream(keytabFile));
+ byte[] keytabContent = null;
+ try {
+ keytabContent = IOUtils.toByteArray(bufferedIn);
+ } finally {
+ bufferedIn.close();
+ }
+ String keytabContentBase64 = Base64.encodeBase64String(keytabContent);
+ keytabMap.put(KerberosServerAction.KEYTAB_CONTENT_BASE64, keytabContentBase64);
+
+ kcp.add(keytabMap);
+ }
+ }
+ } else if (REMOVE_KEYTAB.equalsIgnoreCase(command) || CHECK_KEYTABS.equalsIgnoreCase(command)) {
+ Map<String, String> keytabMap = new HashMap<>();
+ String keytabFilePath = resolvedKeytab.getFile();
+
+ String principal = resolvedPrincipal.getPrincipal();
+ for (Map.Entry<String, String> mappingEntry: resolvedPrincipal.getServiceMapping().entries()) {
+ String serviceName = mappingEntry.getKey();
+ String componentName = mappingEntry.getValue();
keytabMap.put(KerberosIdentityDataFileReader.HOSTNAME, hostName);
- keytabMap.put(KerberosIdentityDataFileReader.SERVICE, isService);
- keytabMap.put(KerberosIdentityDataFileReader.COMPONENT, record.get(KerberosIdentityDataFileReader.COMPONENT));
+ keytabMap.put(KerberosIdentityDataFileReader.SERVICE, serviceName);
+ keytabMap.put(KerberosIdentityDataFileReader.COMPONENT, componentName);
keytabMap.put(KerberosIdentityDataFileReader.PRINCIPAL, principal);
keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH, keytabFilePath);
- keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_NAME, record.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_NAME));
- keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS, record.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS));
- keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_NAME, record.get(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_NAME));
- keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_ACCESS, record.get(KerberosIdentityDataFileReader.KEYTAB_FILE_GROUP_ACCESS));
-
- BufferedInputStream bufferedIn = new BufferedInputStream(new FileInputStream(keytabFile));
- byte[] keytabContent = null;
- try {
- keytabContent = IOUtils.toByteArray(bufferedIn);
- } finally {
- bufferedIn.close();
- }
- String keytabContentBase64 = Base64.encodeBase64String(keytabContent);
- keytabMap.put(KerberosServerAction.KEYTAB_CONTENT_BASE64, keytabContentBase64);
- kcp.add(keytabMap);
}
- }
- } else if (REMOVE_KEYTAB.equalsIgnoreCase(command) || CHECK_KEYTABS.equalsIgnoreCase(command)) {
- Map<String, String> keytabMap = new HashMap<>();
- keytabMap.put(KerberosIdentityDataFileReader.HOSTNAME, hostName);
- keytabMap.put(KerberosIdentityDataFileReader.SERVICE, record.get(KerberosIdentityDataFileReader.SERVICE));
- keytabMap.put(KerberosIdentityDataFileReader.COMPONENT, record.get(KerberosIdentityDataFileReader.COMPONENT));
- keytabMap.put(KerberosIdentityDataFileReader.PRINCIPAL, record.get(KerberosIdentityDataFileReader.PRINCIPAL));
- keytabMap.put(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH, record.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH));
-
- kcp.add(keytabMap);
+ kcp.add(keytabMap);
+ }
}
}
}
} catch (IOException e) {
throw new AmbariException("Could not inject keytabs to enable kerberos");
- } finally {
- if (reader != null) {
- try {
- reader.close();
- } catch (Throwable t) {
- // ignored
- }
- }
}
-
ec.setKerberosCommandParams(kcp);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
index 83d2c98..1374a3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
@@ -53,8 +53,8 @@ import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
-import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.state.Alert;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
@@ -133,10 +133,10 @@ public class HeartbeatProcessor extends AbstractService{
AmbariMetaInfo ambariMetaInfo;
@Inject
- KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
+ KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
@Inject
- KerberosKeytabDAO kerberosKeytabDao;
+ KerberosKeytabDAO kerberosKeytabDAO;
@Inject
Gson gson;
@@ -439,35 +439,32 @@ public class HeartbeatProcessor extends AbstractService{
}
if (writeKeytabsStructuredOut != null) {
+ // TODO rework this. Make sure that keytab check and write commands returns principal list for each keytab
if (SET_KEYTAB.equalsIgnoreCase(customCommand)) {
Map<String, String> keytabs = writeKeytabsStructuredOut.getKeytabs();
if (keytabs != null) {
for (Map.Entry<String, String> entry : keytabs.entrySet()) {
String principal = entry.getKey();
String keytabPath = entry.getValue();
- KerberosPrincipalHostEntity kphe = kerberosPrincipalHostDAO.find(principal, host.getHostId(), keytabPath);
- kphe.setDistributed(true);
- kerberosPrincipalHostDAO.merge(kphe);
+ for (KerberosKeytabPrincipalEntity kkpe: kerberosKeytabPrincipalDAO.findByHostAndKeytab(host.getHostId(), keytabPath)) {
+ kkpe.setDistributed(true);
+ kerberosKeytabPrincipalDAO.merge(kkpe);
+ }
}
}
} else if (REMOVE_KEYTAB.equalsIgnoreCase(customCommand)) {
- Map<String, String> deletedKeytabs = writeKeytabsStructuredOut.getRemovedKeytabs();
- if (deletedKeytabs != null) {
- for (Map.Entry<String, String> entry : deletedKeytabs.entrySet()) {
- String keytabPath = entry.getValue();
- kerberosPrincipalHostDAO.removeByKeytabPath(keytabPath);
- kerberosKeytabDao.remove(keytabPath);
- }
- }
+ // TODO check if additional processing of removed records(besides existent in DestroyPrincipalsServerAction)
+ // TODO is required
}
}
} else if (CHECK_KEYTABS.equalsIgnoreCase(customCommand)) {
ListKeytabsStructuredOut structuredOut = gson.fromJson(report.getStructuredOut(), ListKeytabsStructuredOut.class);
for (MissingKeytab each : structuredOut.missingKeytabs) {
LOG.info("Missing principal: {} for keytab: {} on host: {}", each.principal, each.keytabFilePath, hostname);
- KerberosPrincipalHostEntity kphe = kerberosPrincipalHostDAO.find(each.principal, host.getHostId(), each.keytabFilePath);
- kphe.setDistributed(false);
- kerberosPrincipalHostDAO.merge(kphe);
+ for (KerberosKeytabPrincipalEntity kkpe: kerberosKeytabPrincipalDAO.findByHostAndKeytab(host.getHostId(), each.keytabFilePath)) {
+ kkpe.setDistributed(false);
+ kerberosKeytabPrincipalDAO.merge(kkpe);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
index a7b9d80..9837d70 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
@@ -45,6 +45,7 @@ import org.apache.ambari.server.serveraction.kerberos.DestroyPrincipalsServerAct
import org.apache.ambari.server.serveraction.kerberos.KDCType;
import org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler;
import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.StackId;
@@ -78,7 +79,7 @@ class DeleteIdentityHandler {
public void addDeleteIdentityStages(Cluster cluster, OrderedRequestStageContainer stageContainer, CommandParams commandParameters, boolean manageIdentities)
throws AmbariException
{
- ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent("AMBARI_SERVER", StageUtils.getHostName(), System.currentTimeMillis());
+ ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(RootComponent.AMBARI_SERVER.name(), StageUtils.getHostName(), System.currentTimeMillis());
String hostParamsJson = StageUtils.getGson().toJson(customCommandExecutionHelper.createDefaultHostParams(cluster, cluster.getDesiredStackVersion()));
stageContainer.setClusterHostInfo(StageUtils.getGson().toJson(StageUtils.getClusterHostInfo(cluster)));
if (manageIdentities) {
@@ -321,7 +322,7 @@ class DeleteIdentityHandler {
}
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext) throws AmbariException {
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext) throws AmbariException {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index 749943d..0aef548 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -740,7 +740,7 @@ public interface KerberosHelper {
*
* @param resolvedKerberosKeytab kerberos keytab to be persisted
*/
- void processResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab);
+ void createResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab);
/**
* Removes existent persisted keytabs if they are not in {@code expectedKeytabs} collection.
[39/51] [abbrv] ambari git commit: AMBARI-22676. [Patch Hive]webhcat:
test_sqoop fails with hdfs:///hdp/apps/2.6.*/sqoop/sqoop.tar.gz does not
exist (ncole)
Posted by am...@apache.org.
AMBARI-22676. [Patch Hive]webhcat: test_sqoop fails with hdfs:///hdp/apps/2.6.*/sqoop/sqoop.tar.gz does not exist (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/98356b36
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/98356b36
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/98356b36
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 98356b36b5556735b99248d89b907d957ebef5db
Parents: e404100
Author: Nate Cole <nc...@hortonworks.com>
Authored: Thu Dec 21 20:19:34 2017 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Thu Dec 21 20:19:34 2017 -0500
----------------------------------------------------------------------
.../main/resources/stacks/HDP/2.0.6/properties/stack_packages.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/98356b36/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
index dc71b4d..68da0dd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
@@ -1290,7 +1290,7 @@
},
"upgrade-dependencies" : {
"ATLAS": ["STORM"],
- "HIVE": ["TEZ", "MAPREDUCE2"],
+ "HIVE": ["TEZ", "MAPREDUCE2", "SQOOP"],
"TEZ": ["HIVE"],
"MAPREDUCE2": ["HIVE"],
"MAHOUT": ["MAPREDUCE2"],
[40/51] [abbrv] ambari git commit: AMBARI-22121. Create mpack for
Isilon OneFS (amagyar)
Posted by am...@apache.org.
AMBARI-22121. Create mpack for Isilon OneFS (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d88c8ea4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d88c8ea4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d88c8ea4
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: d88c8ea433e08c1a10e268a3e023f076afcb2cdd
Parents: 98356b3
Author: Attila Magyar <am...@hortonworks.com>
Authored: Fri Oct 6 09:13:09 2017 +0200
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:55:02 2017 +0100
----------------------------------------------------------------------
.../isilon-onefs-mpack/.gitignore | 1 +
.../management-packs/isilon-onefs-mpack/pom.xml | 110 ++++++
.../src/main/assemblies/isilon-onefs-mpack.xml | 40 ++
.../addon-services/ONEFS/1.0.0/alerts.json | 46 +++
.../ONEFS/1.0.0/configuration/core-site.xml | 102 +++++
.../ONEFS/1.0.0/configuration/hadoop-env.xml | 386 +++++++++++++++++++
.../ONEFS/1.0.0/configuration/hdfs-site.xml | 61 +++
.../addon-services/ONEFS/1.0.0/metainfo.xml | 90 +++++
.../addon-services/ONEFS/1.0.0/metrics.json | 59 +++
.../ONEFS/1.0.0/package/scripts/__init__.py | 20 +
.../ONEFS/1.0.0/package/scripts/onefs_client.py | 56 +++
.../ONEFS/1.0.0/package/scripts/params.py | 29 ++
.../ONEFS/1.0.0/package/scripts/params_linux.py | 74 ++++
.../1.0.0/package/scripts/params_windows.py | 83 ++++
.../1.0.0/package/scripts/service_check.py | 137 +++++++
.../1.0.0/package/scripts/status_params.py | 58 +++
.../addon-services/ONEFS/1.0.0/widgets.json | 38 ++
.../src/main/resources/mpack.json | 28 ++
18 files changed, 1418 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/.gitignore
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/.gitignore b/contrib/management-packs/isilon-onefs-mpack/.gitignore
new file mode 100644
index 0000000..1377554
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/.gitignore
@@ -0,0 +1 @@
+*.swp
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/pom.xml b/contrib/management-packs/isilon-onefs-mpack/pom.xml
new file mode 100644
index 0000000..5d8f215
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/pom.xml
@@ -0,0 +1,110 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>isilon-onefs-mpack</artifactId>
+ <packaging>pom</packaging>
+ <version>0.1.0.0-SNAPSHOT</version>
+ <name>Isilon OneFS Ambari Management Pack</name>
+ <url>http://ambari.apache.org/</url>
+ <properties>
+ <minAmbariVersion>3.0.0.0</minAmbariVersion>
+ <maxAmbariVersion></maxAmbariVersion>
+ <nifiversion>1.0.0</nifiversion>
+ </properties>
+ <parent>
+ <groupId>org.apache.ambari.contrib.mpacks</groupId>
+ <artifactId>ambari-contrib-mpacks</artifactId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ </parent>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>parse-version</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>parse-version</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>mpackVersion</name>
+ <value>${project.version}</value>
+ <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
+ <replacement>$1.$2.$3.$4</replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>mpack.json</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptors>
+ <descriptor>src/main/assemblies/isilon-onefs-mpack.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/assemblies/isilon-onefs-mpack.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/assemblies/isilon-onefs-mpack.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/assemblies/isilon-onefs-mpack.xml
new file mode 100644
index 0000000..232cf50
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/assemblies/isilon-onefs-mpack.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id></id>
+ <formats>
+ <format>dir</format>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/addon-services</directory>
+ <outputDirectory>addon-services</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <files>
+ <file>
+ <source>target/mpack.json</source>
+ </file>
+ </files>
+ <dependencySets>
+ </dependencySets>
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
new file mode 100644
index 0000000..5718721
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
@@ -0,0 +1,46 @@
+{
+ "ONEFS":{
+ "service": [
+ {
+ "name": "onefs_namenode_cpu",
+ "label": "OneFS NameNode Host CPU Utilization",
+ "description": "This host-level alert is triggered if CPU utilization of the NameNode exceeds certain warning and critical thresholds. It checks the NameNode JMX Servlet for the SystemCPULoad property. The threshold values are in percent.",
+ "interval": 1,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SERVER",
+ "class": "org.apache.ambari.server.alerts.JmxServerSideAlert",
+ "uri": {
+ "http": "${hdfs-site/dfs.namenode.http-address}",
+ "https": "${hdfs-site/dfs.namenode.https-address}",
+ "https_property": "${hdfs-site/dfs.http.policy}",
+ "https_property_value": "HTTPS_ONLY",
+ "connection_timeout": 5.0
+ },
+ "reporting": {
+ "ok": {
+ "text": "{1} CPU, load {0,number,percent}"
+ },
+ "warning": {
+ "text": "{1} CPU, load {0,number,percent}",
+ "value": 5
+ },
+ "critical": {
+ "text": "{1} CPU, load {0,number,percent}",
+ "value": 20
+ },
+ "units" : "%",
+ "type": "PERCENT"
+ },
+ "jmx": {
+ "property_list": [
+ "java.lang:type=OperatingSystem/SystemCpuLoad",
+ "java.lang:type=OperatingSystem/AvailableProcessors"
+ ]
+ }
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
new file mode 100644
index 0000000..7d3acd7
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!-- Put site-specific property overrides in this file. -->
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
+ <!-- file system properties -->
+ <property>
+ <name>fs.defaultFS</name>
+ <!-- cluster variant -->
+ <value>hdfs://localhost:8020</value>
+ <description>The name of the default file system. Either the
+ literal string "local" or a host:port for HDFS.</description>
+ <final>true</final>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ <description>
+ Set the authentication for the cluster. Valid values are: simple or
+ kerberos.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ <description>
+ Enable authorization for different protocols.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>DEFAULT</value>
+ <description>The mapping from kerberos principal names to local OS mapreduce.job.user.names.
+ So the default rule is just "DEFAULT" which takes all principals in your default domain to their first component.
+ "omalley@APACHE.ORG" and "omalley/admin@APACHE.ORG" to "omalley", if your default domain is APACHE.ORG.
+The translations rules have 3 sections:
+ base filter substitution
+The base consists of a number that represents the number of components in the principal name excluding the realm and the pattern for building the name from the sections of the principal name. The base uses $0 to mean the realm, $1 to mean the first component and $2 to mean the second component.
+
+[1:$1@$0] translates "omalley@APACHE.ORG" to "omalley@APACHE.ORG"
+[2:$1] translates "omalley/admin@APACHE.ORG" to "omalley"
+[2:$1%$2] translates "omalley/admin@APACHE.ORG" to "omalley%admin"
+
+The filter is a regex in parens that must the generated string for the rule to apply.
+
+"(.*%admin)" will take any string that ends in "%admin"
+"(.*@ACME.COM)" will take any string that ends in "@ACME.COM"
+
+Finally, the substitution is a sed rule to translate a regex into a fixed string.
+
+"s/@ACME\.COM//" removes the first instance of "@ACME.COM".
+"s/@[A-Z]*\.COM//" removes the first instance of "@" followed by a name followed by ".COM".
+"s/X/Y/g" replaces all of the "X" in the name with "Y"
+
+So, if your default realm was APACHE.ORG, but you also wanted to take all principals from ACME.COM that had a single component "joe@ACME.COM", you'd do:
+
+RULE:[1:$1@$0](.@ACME.ORG)s/@.//
+DEFAULT
+
+To also translate the names with a second component, you'd make the rules:
+
+RULE:[1:$1@$0](.@ACME.ORG)s/@.//
+RULE:[2:$1@$0](.@ACME.ORG)s/@.//
+DEFAULT
+
+If you want to treat all principals from APACHE.ORG with /admin as "admin", your rules would look like:
+
+RULE[2:$1%$2@$0](.%admin@APACHE.ORG)s/./admin/
+DEFAULT
+ </description>
+ <value-attributes>
+ <type>multiLine</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>net.topology.script.file.name</name>
+ <value>/etc/hadoop/conf/topology_script.py</value>
+ <description>
+ Location of topology script used by Hadoop to determine the rack location of nodes.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
new file mode 100644
index 0000000..bb671cc
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
@@ -0,0 +1,386 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>hdfs_log_dir_prefix</name>
+ <value>/var/log/hadoop</value>
+ <description>Hadoop Log Dir Prefix</description>
+ <display-name>Hadoop Log Dir Prefix</display-name>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop_pid_dir_prefix</name>
+ <value>/var/run/hadoop</value>
+ <display-name>Hadoop PID Dir Prefix</display-name>
+ <description>Hadoop PID Dir Prefix</description>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop_root_logger</name>
+ <value>INFO,RFA</value>
+ <display-name>Hadoop Root Logger</display-name>
+ <description>Hadoop Root Logger</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop_heapsize</name>
+ <value>1024</value>
+ <description>Hadoop maximum Java heap size</description>
+ <display-name>Hadoop maximum Java heap size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>namenode_heapsize</name>
+ <value>1024</value>
+ <description>NameNode Java heap size</description>
+ <display-name>NameNode Java heap size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>268435456</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ <overridable>false</overridable>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hdfs-site</type>
+ <name>dfs.datanode.data.dir</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>namenode_opt_newsize</name>
+ <value>200</value>
+ <description>Default size of Java new generation for NameNode (Java option -XX:NewSize) Note: The value of NameNode new generation size (default size of Java new generation for NameNode (Java option -XX:NewSize)) should be 1/8 of maximum heap size (-Xmx). Ensure that the value of the namenode_opt_newsize property is 1/8 the value of maximum heap size (-Xmx).</description>
+ <display-name>NameNode new generation size</display-name>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>namenode_heapsize</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>16384</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>namenode_opt_maxnewsize</name>
+ <value>200</value>
+ <description>NameNode maximum new generation size</description>
+ <display-name>NameNode maximum new generation size</display-name>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>namenode_heapsize</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>16384</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>namenode_opt_permsize</name>
+ <value>128</value>
+ <description>NameNode permanent generation size</description>
+ <display-name>NameNode permanent generation size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>2096</maximum>
+ <unit>MB</unit>
+ <increment-step>128</increment-step>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>namenode_opt_maxpermsize</name>
+ <value>256</value>
+ <description>NameNode maximum permanent generation size</description>
+ <display-name>NameNode maximum permanent generation size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>2096</maximum>
+ <unit>MB</unit>
+ <increment-step>128</increment-step>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dtnode_heapsize</name>
+ <value>1024</value>
+ <description>DataNode maximum Java heap size</description>
+ <display-name>DataNode maximum Java heap size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>268435456</maximum>
+ <unit>MB</unit>
+ <increment-step>128</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>proxyuser_group</name>
+ <display-name>Proxy User Group</display-name>
+ <value>users</value>
+ <property-type>GROUP</property-type>
+ <description>Proxy user group.</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hdfs_user</name>
+ <display-name>HDFS User</display-name>
+ <value>hdfs</value>
+ <property-type>USER</property-type>
+ <description>User to run HDFS as</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hdfs_tmp_dir</name>
+ <value>/tmp</value>
+ <description>HDFS tmp Dir</description>
+ <display-name>HDFS tmp Dir</display-name>
+ <property-type>NOT_MANAGED_HDFS_PATH</property-type>
+ <value-attributes>
+ <read-only>true</read-only>
+ <overridable>false</overridable>
+ <visible>false</visible>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hdfs_user_nofile_limit</name>
+ <value>128000</value>
+ <description>Max open files limit setting for HDFS user.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hdfs_user_nproc_limit</name>
+ <value>65536</value>
+ <description>Max number of processes limit setting for HDFS user.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>namenode_backup_dir</name>
+ <description>Local directory for storing backup copy of NameNode images during upgrade</description>
+ <value>/tmp/upgrades</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hdfs_user_keytab</name>
+ <description>HDFS keytab path</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hdfs_principal_name</name>
+ <description>HDFS principal name</description>
+ <property-type>KERBEROS_PRINCIPAL</property-type>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <!-- hadoop-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>hadoop-env template</display-name>
+ <description>This is the jinja template for hadoop-env.sh file</description>
+ <value>
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME. All others are
+# optional. When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use. Required.
+export JAVA_HOME={{java_home}}
+export HADOOP_HOME_WARN_SUPPRESS=1
+
+# Hadoop home directory
+export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
+
+# Hadoop Configuration Directory
+#TODO: if env var set that can cause problems
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}
+
+
+# Path to jsvc required by secure datanode
+export JSVC_HOME={{jsvc_path}}
+
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+if [[ ("$SERVICE" = "hiveserver2") || ("$SERVICE" = "metastore") || ( "$SERVICE" = "cli") ]]; then
+ if [ "$HADOOP_HEAPSIZE" = "" ]; then
+ export HADOOP_HEAPSIZE="{{hadoop_heapsize}}"
+ fi
+else
+ export HADOOP_HEAPSIZE="{{hadoop_heapsize}}"
+fi
+
+
+export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}"
+
+# Extra Java runtime options. Empty by default.
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
+
+# Command specific options appended to HADOOP_OPTS when specified
+
+{% if java_version < 8 %}
+export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}"
+export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS"
+{% else %}
+export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}"
+export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
+{% endif %}
+HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
+
+HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
+HADOOP_DATANODE_OPTS="-XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:ConcGCThreads=4 -XX:+UseConcMarkSweepGC -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
+HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}"
+
+# On secure datanodes, user to run the datanode as after dropping privileges
+export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}
+
+# Extra ssh options. Empty by default.
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
+
+# Where log files are stored. $HADOOP_HOME/logs by default.
+export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER
+
+# History server logs
+export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER
+
+# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.
+# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
+
+# host:path where hadoop code should be rsync'd from. Unset by default.
+# export HADOOP_MASTER=master:/home/$USER/src/hadoop
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HADOOP_SLAVE_SLEEP=0.1
+
+# The directory where pid files are stored. /tmp by default.
+export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER
+export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER
+
+# History server pid
+export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER
+
+YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY -Drm.audit.logger=INFO,RMAUDIT"
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+
+# export HADOOP_NICENESS=10
+
+# Add database libraries
+JAVA_JDBC_LIBS=""
+if [ -d "/usr/share/java" ]; then
+ for jarFile in `ls /usr/share/java | grep -E "(mysql|ojdbc|postgresql|sqljdbc)" 2>/dev/null`
+ do
+ JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
+ done
+fi
+
+# Add libraries required by nodemanager
+MAPREDUCE_LIBS={{mapreduce_libs_path}}
+
+# Add libraries to the hadoop classpath - some may not need a colon as they already include it
+export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
+
+if [ -d "/usr/lib/tez" ]; then
+ export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf
+fi
+
+# Setting path to hdfs command line
+export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
+
+#Mostly required for hadoop 2.0
+export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64
+
+{% if is_datanode_max_locked_memory_set %}
+# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin.
+# Makes sense to fix only when runing DN as root
+if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ ulimit -l {{datanode_max_locked_memory}}
+fi
+{% endif %}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
new file mode 100644
index 0000000..cb6544f
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration supports_final="true">
+ <property>
+ <name>dfs.webhdfs.enabled</name>
+ <value>false</value>
+ <display-name>WebHDFS enabled</display-name>
+ <description>Whether to enable WebHDFS feature</description>
+ <final>true</final>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:8082</value>
+ <description>
+ The datanode http server address and port.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.namenode.http-address</name>
+ <value>localhost:8082</value>
+ <description>The name of the default file system. Either the
+ literal string "local" or a host:port for HDFS.</description>
+ <final>true</final>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>dfs.namenode.https-address</name>
+ <value>localhost:8080</value>
+ <description>The https address where namenode binds</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>dfs.client-write-packet-size</name>
+ <value>131072</value>
+ <description>Packet size for clients to write</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
new file mode 100644
index 0000000..f20bcf8
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ONEFS</name>
+ <displayName>OneFS</displayName>
+ <serviceType>HCFS</serviceType>
+ <comment>Isilon Systems OneFS</comment>
+ <version>1.0.0</version>
+ <components>
+ <component>
+ <name>ONEFS_CLIENT</name>
+ <displayName>OneFS Client</displayName>
+ <category>CLIENT</category>
+ <componentType>HCFS_CLIENT</componentType>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <commandScript>
+ <script>scripts/onefs_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>hdfs-site.xml</fileName>
+ <dictionaryName>hdfs-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>core-site.xml</fileName>
+ <dictionaryName>core-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>log4j.properties</fileName>
+ <dictionaryName>hdfs-log4j,yarn-log4j</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hadoop-env.sh</fileName>
+ <dictionaryName>hadoop-env</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>hadoop-client</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <configuration-dependencies>
+ <config-type>core-site</config-type>
+ <config-type>hdfs-site</config-type>
+ <config-type>hadoop-env</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
new file mode 100644
index 0000000..cd705eb
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
@@ -0,0 +1,59 @@
+{
+ "ONEFS_CLIENT": {
+ "Component": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "default": {
+ "metrics/onefs/demo/counter": {
+ "metric": "onefs.demo.counter",
+ "pointInTime": true,
+ "temporal": true
+ }
+ }
+ }
+ },
+ {
+ "type": "jmx",
+ "properties": {
+ "url_config_type": "hdfs-site",
+ "url_property_name": "dfs.namenode.http-address"
+ },
+ "metrics": {
+ "default": {
+ "metrics/dfs/namenode/Used": {
+ "metric": "Hadoop:service=NameNode,name=NameNodeInfo.Used",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemHeapUsedM",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/HeapMemoryMax": {
+ "metric": "java.lang:type=Memory.HeapMemoryUsage[max]",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/HeapMemoryUsed": {
+ "metric": "java.lang:type=Memory.HeapMemoryUsage[used]",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/NonHeapMemoryMax": {
+ "metric": "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/NonHeapMemoryUsed": {
+ "metric": "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+ "pointInTime": true,
+ "temporal": false
+ }
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/__init__.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/__init__.py
new file mode 100644
index 0000000..35de4bb
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/__init__.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/onefs_client.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/onefs_client.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/onefs_client.py
new file mode 100644
index 0000000..dbf1331
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/onefs_client.py
@@ -0,0 +1,56 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.libraries.script import Script
+from resource_management.libraries.resources.xml_config import XmlConfig
+
+class OneFsClient(Script):
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ self.setup_config(env)
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+ def setup_config(self, env):
+ import params
+ env.set_params(params)
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ XmlConfig("core-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+if __name__ == "__main__":
+ OneFsClient().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params.py
new file mode 100644
index 0000000..838510c
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params.py
@@ -0,0 +1,29 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize']
+retryAble = default("/commandParams/command_retry_enabled", False)
+script_https_protocol = Script.get_force_https_protocol_name()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..72e0ae9
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
@@ -0,0 +1,74 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.functions.version import format_stack_version, compare_versions
+from resource_management import *
+import os
+import itertools
+import re
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+
+config = Script.get_config()
+
+hostname = config["hostname"]
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
+hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", [])
+zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", [])
+
+has_journalnode_hosts = not len(journalnode_hosts) == 0
+has_zkfc_hosts = not len(zkfc_hosts) == 0
+is_namenode_master = hostname in namenode_host
+
+dfs_type = default("/commandParams/dfs_type", "")
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None)
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_windows.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..3e712b3
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_windows.py
@@ -0,0 +1,83 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+#Used in subsequent imports from params
+from install_params import exclude_packages
+from status_params import *
+
+config = Script.get_config()
+hadoop_conf_dir = None
+hbase_conf_dir = None
+hadoop_home = None
+try:
+ hadoop_conf_dir = os.environ["HADOOP_CONF_DIR"]
+ hbase_conf_dir = os.environ["HBASE_CONF_DIR"]
+ hadoop_home = os.environ["HADOOP_HOME"]
+except:
+ pass
+#directories & files
+dfs_name_dir = config['configurations']['hdfs-site']['dfs.namenode.name.dir']
+fs_checkpoint_dir = config['configurations']['hdfs-site']['dfs.namenode.checkpoint.dir']
+dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir']
+#decomission
+hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
+exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
+include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
+hdfs_include_file = None
+manage_include_files = default("/configurations/hdfs-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+ hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file))
+update_files_only = default("/commandParams/update_files_only",False)
+# HDFS High Availability properties
+dfs_ha_enabled = False
+dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None)
+dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None)
+
+namenode_id = None
+namenode_rpc = None
+hostname = config["hostname"]
+if dfs_ha_namenode_ids:
+ dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",")
+ dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list)
+ if dfs_ha_namenode_ids_array_len > 1:
+ dfs_ha_enabled = True
+if dfs_ha_enabled:
+ for nn_id in dfs_ha_namemodes_ids_list:
+ nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')]
+ if hostname.lower() in nn_host.lower():
+ namenode_id = nn_id
+ namenode_rpc = nn_host
+
+hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+hdfs_user = hadoop_user
+
+grep_exe = "findstr"
+
+name_node_params = default("/commandParams/namenode", None)
+
+service_map = {
+ "datanode" : datanode_win_service_name,
+ "journalnode" : journalnode_win_service_name,
+ "namenode" : namenode_win_service_name,
+ "secondarynamenode" : snamenode_win_service_name,
+ "zkfc_slave": zkfc_win_service_name
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
new file mode 100644
index 0000000..3d798a3
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
@@ -0,0 +1,137 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.core.shell import as_user
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons import OSConst
+from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from resource_management.core.logger import Logger
+
+class HdfsServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class HdfsServiceCheckDefault(HdfsServiceCheck):
+ def service_check(self, env):
+ import params
+
+ env.set_params(params)
+ unique = functions.get_unique_id_and_date()
+ dir = params.hdfs_tmp_dir
+ tmp_file = format("{dir}/{unique}")
+
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
+ user=params.hdfs_user
+ )
+ params.HdfsResource(dir,
+ type="directory",
+ action="create_on_execute",
+ mode=0777
+ )
+ params.HdfsResource(tmp_file,
+ type="file",
+ action="delete_on_execute",
+ )
+
+ params.HdfsResource(tmp_file,
+ type="file",
+ source="/etc/passwd",
+ action="create_on_execute"
+ )
+ params.HdfsResource(None, action="execute")
+
+ if params.has_journalnode_hosts:
+ if params.security_enabled:
+ for host in params.journalnode_hosts:
+ if params.https_only:
+ uri = format("https://{host}:{journalnode_port}")
+ else:
+ uri = format("http://{host}:{journalnode_port}")
+ response, errmsg, time_millis = curl_krb_request(params.tmp_dir, params.smoke_user_keytab,
+ params.smokeuser_principal, uri, "jn_service_check",
+ params.kinit_path_local, False, None, params.smoke_user)
+ if not response:
+ Logger.error("Cannot access WEB UI on: {0}. Error : {1}", uri, errmsg)
+ return 1
+ else:
+ journalnode_port = params.journalnode_port
+ checkWebUIFileName = "checkWebUI.py"
+ checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
+ comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
+
+ checkWebUICmd = format("ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only} -o {script_https_protocol}")
+ File(checkWebUIFilePath,
+ content=StaticFile(checkWebUIFileName),
+ mode=0775)
+
+ Execute(checkWebUICmd,
+ logoutput=True,
+ try_sleep=3,
+ tries=5,
+ user=params.smoke_user
+ )
+
+ if params.is_namenode_master:
+ if params.has_zkfc_hosts:
+ pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
+ pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid")
+ check_zkfc_process_cmd = as_user(format(
+ "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.hdfs_user)
+ Execute(check_zkfc_process_cmd,
+ logoutput=True,
+ try_sleep=3,
+ tries=5
+ )
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class HdfsServiceCheckWindows(HdfsServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ unique = functions.get_unique_id_and_date()
+
+ #Hadoop uses POSIX-style paths, separator is always /
+ dir = params.hdfs_tmp_dir
+ tmp_file = dir + '/' + unique
+
+ #commands for execution
+ hadoop_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hadoop.cmd"))
+ create_dir_cmd = "%s fs -mkdir %s" % (hadoop_cmd, dir)
+ own_dir = "%s fs -chmod 777 %s" % (hadoop_cmd, dir)
+ test_dir_exists = "%s fs -test -e %s" % (hadoop_cmd, dir)
+ cleanup_cmd = "%s fs -rm %s" % (hadoop_cmd, tmp_file)
+ create_file_cmd = "%s fs -put %s %s" % (hadoop_cmd, os.path.join(params.hadoop_conf_dir, "core-site.xml"), tmp_file)
+ test_cmd = "%s fs -test -e %s" % (hadoop_cmd, tmp_file)
+
+ hdfs_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hdfs.cmd"))
+ safemode_command = "%s dfsadmin -safemode get | %s OFF" % (hdfs_cmd, params.grep_exe)
+
+ Execute(safemode_command, logoutput=True, try_sleep=3, tries=20)
+ Execute(create_dir_cmd, user=params.hdfs_user,logoutput=True, ignore_failures=True)
+ Execute(own_dir, user=params.hdfs_user,logoutput=True)
+ Execute(test_dir_exists, user=params.hdfs_user,logoutput=True)
+ Execute(create_file_cmd, user=params.hdfs_user,logoutput=True)
+ Execute(test_cmd, user=params.hdfs_user,logoutput=True)
+ Execute(cleanup_cmd, user=params.hdfs_user,logoutput=True)
+
+if __name__ == "__main__":
+ HdfsServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/status_params.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/status_params.py
new file mode 100644
index 0000000..153f9a6
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/status_params.py
@@ -0,0 +1,58 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons import OSCheck
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.script.script import Script
+
+config = Script.get_config()
+
+if OSCheck.is_windows_family():
+ namenode_win_service_name = "namenode"
+ datanode_win_service_name = "datanode"
+ snamenode_win_service_name = "secondarynamenode"
+ journalnode_win_service_name = "journalnode"
+ zkfc_win_service_name = "zkfc"
+else:
+ hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+ hadoop_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
+ datanode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-datanode.pid")
+ namenode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-namenode.pid")
+ snamenode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-secondarynamenode.pid")
+ journalnode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-journalnode.pid")
+ zkfc_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-zkfc.pid")
+ nfsgateway_pid_file = format("{hadoop_pid_dir_prefix}/root/hadoop_privileged_nfs3.pid")
+
+ # Security related/required params
+ hostname = config['hostname']
+ security_enabled = config['configurations']['cluster-env']['security_enabled']
+ hdfs_user_principal = config['configurations']['hadoop-env']['hdfs_principal_name']
+ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+
+ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+ tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
new file mode 100644
index 0000000..23da1d6
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
@@ -0,0 +1,38 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_onefs_layout",
+ "display_name": "Standard ONEFS Dashboard",
+ "section_name": "ONEFS_SUMMARY",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "OneFS metrics demo",
+ "description": "Test widget",
+ "default_section_name": "ONEFS_SUMMARY",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "scope" : "SERVICE",
+ "metrics": [
+ {
+ "name": "counter",
+ "metric_path": "metrics/onefs/demo/counter",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLUSTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "demo value",
+ "values" : "${counter}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1",
+ "display_unit": "%"
+ }
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d88c8ea4/contrib/management-packs/isilon-onefs-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/mpack.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..f15fcb6
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/mpack.json
@@ -0,0 +1,28 @@
+{
+ "type" : "full-release",
+ "name" : "onefs-ambari-mpack",
+ "version": "0.1",
+ "description" : "OneFS Ambari Management Pack",
+ "prerequisites": {
+ "min-ambari-version" : "3.0.0.0"
+ },
+ "artifacts": [
+ {
+ "name" : "ONEFS-addon-services",
+ "type" : "stack-addon-service-definitions",
+ "source_dir" : "addon-services",
+ "service_versions_map" : [
+ {
+ "service_name" : "ONEFS",
+ "service_version" : "1.0.0",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP", "stack_version" : "2.6"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
+
[12/51] [abbrv] ambari git commit: AMBARI-22647. Rafactor: Package
Log Search and Log Feeder into jars + cleanup start scripts (oleewere)
Posted by am...@apache.org.
AMBARI-22647. Rafactor: Package Log Search and Log Feeder into jars + cleanup start scripts (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/562795c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/562795c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/562795c4
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 562795c4d35bf3b45d196856e9d66e5dfdd0a17b
Parents: f9ad13d
Author: Oliver Szabo <ol...@gmail.com>
Authored: Fri Dec 15 22:17:46 2017 +0100
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Mon Dec 18 16:28:38 2017 +0100
----------------------------------------------------------------------
.../ambari-logsearch-assembly/pom.xml | 117 +++-----
.../src/main/package/deb/logfeeder/postinst | 22 ++
.../src/main/package/deb/logfeeder/postrm | 6 +
.../src/main/package/deb/logfeeder/posttrm | 15 -
.../src/main/package/deb/logfeeder/preinst | 13 +
.../src/main/package/deb/portal/postinst | 19 ++
.../src/main/package/deb/portal/postrm | 6 +
.../src/main/package/deb/portal/preinst | 13 +
.../main/package/rpm/logfeeder/postinstall.sh | 37 +++
.../main/package/rpm/logfeeder/postremove.sh | 21 ++
.../main/package/rpm/logfeeder/preinstall.sh | 28 ++
.../src/main/package/rpm/portal/postinstall.sh | 34 +++
.../src/main/package/rpm/portal/postremove.sh | 21 ++
.../src/main/package/rpm/portal/preinstall.sh | 28 ++
.../ambari-logsearch-logfeeder/README.md | 4 +-
.../ambari-logsearch-logfeeder/build.properties | 18 --
.../ambari-logsearch-logfeeder/build.xml | 28 +-
.../ambari-logsearch-logfeeder/pom.xml | 31 --
.../ambari-logsearch-logfeeder/run.sh | 4 +-
.../org/apache/ambari/logfeeder/LogFeeder.java | 35 +--
.../ambari/logfeeder/LogFeederCommandLine.java | 63 ++--
.../ambari/logfeeder/common/ConfigHandler.java | 7 +-
.../src/main/package/deb/control/control | 22 --
.../src/main/package/deb/control/postinst | 15 -
.../src/main/package/deb/control/postrm | 15 -
.../src/main/package/deb/control/preinst | 15 -
.../src/main/package/deb/control/prerm | 15 -
.../src/main/scripts/logfeeder-env.sh | 40 +++
.../src/main/scripts/logfeeder.sh | 292 +++++++++++++++++++
.../src/main/scripts/run.sh | 103 -------
.../ambari-logsearch-server/README.md | 2 +-
.../ambari-logsearch-server/build.xml | 36 ++-
.../ambari-logsearch-server/pom.xml | 11 -
ambari-logsearch/ambari-logsearch-server/run.sh | 2 +-
.../org/apache/ambari/logsearch/LogSearch.java | 2 +-
.../src/main/scripts/logsearch-env.sh | 40 +++
.../src/main/scripts/logsearch.sh | 274 +++++++++++++++++
.../src/main/scripts/run.sh | 99 -------
.../src/main/scripts/stop.sh | 44 ---
ambari-logsearch/docker/Dockerfile | 6 +-
ambari-logsearch/docker/all.yml | 4 +-
ambari-logsearch/docker/bin/start.sh | 30 +-
ambari-logsearch/docker/docker-compose.yml | 2 -
ambari-logsearch/docker/logsearch-logfeeder.yml | 1 -
ambari-logsearch/docker/logsearch-server.yml | 1 -
.../test-config/logfeeder/logfeeder-env.sh | 5 +-
.../test-config/logfeeder/logfeeder.properties | 2 +-
.../test-config/logsearch/logsearch-env.sh | 5 +-
.../logsearch/logsearch-https.properties | 4 +-
.../test-config/logsearch/logsearch.properties | 4 +-
.../0.5.0/configuration/logfeeder-env.xml | 2 +-
.../configuration/logfeeder-properties.xml | 2 +-
.../0.5.0/configuration/logsearch-env.xml | 6 +-
.../0.5.0/package/scripts/logfeeder.py | 7 +-
.../0.5.0/package/scripts/logsearch.py | 8 +-
.../0.5.0/package/scripts/logsearch_common.py | 54 ----
.../LOGSEARCH/0.5.0/package/scripts/params.py | 8 +-
.../0.5.0/package/scripts/setup_logfeeder.py | 2 +-
.../0.5.0/package/scripts/setup_logsearch.py | 2 +-
.../0.5.0/properties/logfeeder-env.sh.j2 | 5 +-
.../0.5.0/properties/logsearch-env.sh.j2 | 5 +-
.../stack-hooks/after-INSTALL/scripts/params.py | 2 +-
.../hooks/after-INSTALL/test_after_install.py | 8 +-
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 34 +--
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 31 +-
.../test/python/stacks/2.4/configs/default.json | 2 +-
66 files changed, 1102 insertions(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/pom.xml b/ambari-logsearch/ambari-logsearch-assembly/pom.xml
index cbc62ce..19c3d90 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-assembly/pom.xml
@@ -82,6 +82,18 @@
<configuration>
<group>Development</group>
<name>${logsearch.portal.package.name}</name>
+ <postinstallScriptlet>
+ <scriptFile>${project.basedir}/src/main/package/rpm/portal/postinstall.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </postinstallScriptlet>
+ <postremoveScriptlet>
+ <scriptFile>${project.basedir}/src/main/package/rpm/portal/postremove.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </postremoveScriptlet>
+ <preinstallScriptlet>
+ <scriptFile>${project.basedir}/src/main/package/rpm/portal/preinstall.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </preinstallScriptlet>
<mappings>
<mapping>
<directory>${logsearch.portal.mapping.path}</directory>
@@ -89,37 +101,11 @@
<source>
<location>${logsearch.server.dir}/target/package</location>
<excludes>
- <exclude>classes/log4j.xml</exclude>
- <exclude>classes/logsearch.properties</exclude>
- <exclude>classes/user_pass.json</exclude>
- <exclude>classes/HadoopServiceConfig.json</exclude>
- <exclude>solr_configsets/**</exclude>
<exclude>libs/checkstyle*.jar</exclude>
</excludes>
</source>
</sources>
</mapping>
- <mapping>
- <directory>${logsearch.portal.conf.mapping.path}</directory>
- <sources>
- <source>
- <location>${logsearch.server.dir}/target/package/classes</location>
- <includes>
- <include>log4j.xml</include>
- <include>logsearch.properties</include>
- <include>user_pass.json</include>
- </includes>
- </source>
- </sources>
- </mapping>
- <mapping>
- <directory>${logsearch.portal.conf.mapping.path}/solr_configsets</directory>
- <sources>
- <source>
- <location>${logsearch.server.dir}/target/package/solr_configsets</location>
- </source>
- </sources>
- </mapping>
</mappings>
</configuration>
</execution>
@@ -132,6 +118,18 @@
<configuration>
<group>Development</group>
<name>${logsearch.logfeeder.package.name}</name>
+ <postinstallScriptlet>
+ <scriptFile>${project.basedir}/src/main/package/rpm/logfeeder/postinstall.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </postinstallScriptlet>
+ <postremoveScriptlet>
+ <scriptFile>${project.basedir}/src/main/package/rpm/logfeeder/postremove.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </postremoveScriptlet>
+ <preinstallScriptlet>
+ <scriptFile>${project.basedir}/src/main/package/rpm/logfeeder/preinstall.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </preinstallScriptlet>
<mappings>
<mapping>
<directory>${logsearch.logfeeder.mapping.path}</directory>
@@ -139,27 +137,11 @@
<source>
<location>${logsearch.logfeeder.dir}/target/package</location>
<excludes>
- <exclude>classes/log4j.xml</exclude>
- <exclude>classes/logfeeder.properties</exclude>
- <exclude>classes/grok-patterns</exclude>
<exclude>libs/checkstyle*.jar</exclude>
</excludes>
</source>
</sources>
</mapping>
- <mapping>
- <directory>${logsearch.logfeeder.conf.mapping.path}</directory>
- <sources>
- <source>
- <location>${logsearch.logfeeder.dir}/target/package/classes</location>
- <includes>
- <include>log4j.xml</include>
- <include>logfeeder.properties</include>
- <include>grok-patterns</include>
- </includes>
- </source>
- </sources>
- </mapping>
</mappings>
</configuration>
</execution>
@@ -197,7 +179,11 @@
<directory>${project.basedir}/src/main/package/deb</directory>
<excludes>
<exclude>portal/postinst</exclude>
+ <exclude>portal/postrm</exclude>
+ <exclude>portal/preinstall</exclude>
<exclude>logfeeder/postinst</exclude>
+ <exclude>logfeeder/postrm</exclude>
+ <exclude>logfeeder/preinstall</exclude>
</excludes>
<filtering>false</filtering>
</resource>
@@ -205,7 +191,11 @@
<directory>${project.basedir}/src/main/package/deb</directory>
<includes>
<include>portal/postinst</include>
+ <include>portal/postrm</include>
+ <include>portal/preinstall</include>
<include>logfeeder/postinst</include>
+ <include>logfeeder/postrm</include>
+ <include>logfeeder/preinstall</include>
</includes>
<filtering>true</filtering>
</resource>
@@ -242,34 +232,9 @@
<group>root</group>
</mapper>
<excludes>
- classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,classes/HadoopServiceConfig.json,solr_configsets/**,libs/checkstyle*.jar
+ libs/checkstyle*.jar
</excludes>
</data>
- <data>
- <src>${logsearch.server.dir}/target/package/classes</src>
- <type>directory</type>
- <mapper>
- <prefix>${logsearch.portal.conf.mapping.path}</prefix>
- <type>perm</type>
- <user>root</user>
- <group>root</group>
- <filemode>644</filemode>
- </mapper>
- <includes>
- log4j.xml,logsearch.properties,user_pass.json
- </includes>
- </data>
- <data>
- <src>${logsearch.server.dir}/target/package/solr_configsets</src>
- <type>directory</type>
- <mapper>
- <prefix>${logsearch.portal.conf.mapping.path}/solr_configsets</prefix>
- <type>perm</type>
- <user>root</user>
- <group>root</group>
- </mapper>
- <excludes>libs/checkstyle*.jar</excludes>
- </data>
</dataSet>
</configuration>
</execution>
@@ -297,23 +262,9 @@
<group>root</group>
</mapper>
<excludes>
- classes/log4j.xml,classes/logfeeder.properties,classes/grok-patterns,libs/checkstyle*.jar
+ libs/checkstyle*.jar
</excludes>
</data>
- <data>
- <src>${logsearch.logfeeder.dir}/target/package/classes</src>
- <type>directory</type>
- <mapper>
- <prefix>${logsearch.logfeeder.conf.mapping.path}</prefix>
- <type>perm</type>
- <user>root</user>
- <group>root</group>
- <filemode>644</filemode>
- </mapper>
- <includes>
- log4j.xml,logfeeder.properties,grok-patterns
- </includes>
- </data>
</dataSet>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
index 21a01fa..a4eec77 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
@@ -13,3 +13,25 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
+
+LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder"
+LOGFEEDER_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-logfeeder/bin/logfeeder.sh"
+
+LOGFEEDER_CONF_LINK="/etc/ambari-logsearch-logfeeder/conf"
+LOGFEEDER_CONF_SOURCE="/usr/lib/ambari-logsearch-logfeeder/conf"
+
+ln -s $LOGFEEDER_SCRIPT_SOURCE $LOGFEEDER_SCRIPT_LINK_NAME
+ln -s $LOGFEEDER_CONF_SOURCE $LOGFEEDER_CONF_LINK
+
+# handle old checkpoint & keys folder
+
+LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old"
+
+if [ -d "$LOGFEEDER_CONF_BACKUP" ]; then
+ if [ -d "$LOGFEEDER_CONF_BACKUP/keys" ]; then
+ cp -r $LOGFEEDER_CONF_BACKUP/keys $LOGFEEDER_CONF_SOURCE
+ fi
+ if [ -d "$LOGFEEDER_CONF_BACKUP/checkpoints" ]; then
+ cp -r $LOGFEEDER_CONF_BACKUP/checkpoints $LOGFEEDER_CONF_SOURCE
+ fi
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm
index 21a01fa..5f1c623 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm
@@ -13,3 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
+
+LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder"
+LOGFEEDER_CONF_DIR_LINK="/etc/ambari-logsearch-logfeeder/conf"
+
+rm -f $LOGFEEDER_SCRIPT_LINK_NAME
+rm -f $LOGFEEDER_CONF_DIR_LINK
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm
deleted file mode 100644
index 21a01fa..0000000
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst
index 21a01fa..01d70e0 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst
@@ -13,3 +13,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
+
+LOGFEEDER_CONF="/etc/ambari-logsearch-logfeeder/conf"
+LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old"
+
+if [ -d $LOGFEEDER_CONF_BACKUP ]; then
+ rm -rf $LOGFEEDER_CONF_BACKUP
+fi
+
+if [ -d $LOGFEEDER_CONF ]; then
+ mkdir -p $LOGFEEDER_CONF_BACKUP
+ cp -r $LOGFEEDER_CONF/* $LOGFEEDER_CONF_BACKUP
+ rm -rf $LOGFEEDER_CONF
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst
index 21a01fa..9107482 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst
@@ -13,3 +13,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
+
+LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch"
+LOGSEARCH_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-portal/bin/logsearch.sh"
+
+LOGSEARCH_CONF_LINK="/etc/ambari-logsearch-portal/conf"
+LOGSEARCH_CONF_SOURCE="/usr/lib/ambari-logsearch-portal/conf"
+
+ln -s $LOGSEARCH_SCRIPT_SOURCE $LOGSEARCH_SCRIPT_LINK_NAME
+ln -s $LOGSEARCH_CONF_SOURCE $LOGSEARCH_CONF_LINK
+
+# handle old keys folder
+
+LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old"
+
+if [ -d "$LOGSEARCH_CONF_BACKUP" ]; then
+ if [ -d "$LOGSEARCH_CONF_BACKUP/keys" ]; then
+ cp -r $LOGSEARCH_CONF_BACKUP/keys $LOGSEARCH_CONF_SOURCE
+ fi
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm
index 21a01fa..8de05c1 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm
@@ -13,3 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
+
+LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch"
+LOGSEARCH_CONF_DIR_LINK="/etc/ambari-logsearch-portal/conf"
+
+rm -f $LOGSEARCH_SCRIPT_LINK_NAME
+rm -f $LOGSEARCH_CONF_DIR_LINK
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst
index 21a01fa..dfb8752 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst
@@ -13,3 +13,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
+
+LOGSEARCH_CONF="/etc/ambari-logsearch-portal/conf"
+LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old"
+
+if [ -d $LOGSEARCH_CONF_BACKUP ]; then
+ rm -rf $LOGSEARCH_CONF_BACKUP
+fi
+
+if [ -d $LOGSEARCH_CONF ]; then
+ mkdir -p $LOGSEARCH_CONF_BACKUP
+ cp -r $LOGSEARCH_CONF/* $LOGSEARCH_CONF_BACKUP
+ rm -rf $LOGSEARCH_CONF
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh
new file mode 100755
index 0000000..a4eec77
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder"
+LOGFEEDER_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-logfeeder/bin/logfeeder.sh"
+
+LOGFEEDER_CONF_LINK="/etc/ambari-logsearch-logfeeder/conf"
+LOGFEEDER_CONF_SOURCE="/usr/lib/ambari-logsearch-logfeeder/conf"
+
+ln -s $LOGFEEDER_SCRIPT_SOURCE $LOGFEEDER_SCRIPT_LINK_NAME
+ln -s $LOGFEEDER_CONF_SOURCE $LOGFEEDER_CONF_LINK
+
+# handle old checkpoint & keys folder
+
+LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old"
+
+if [ -d "$LOGFEEDER_CONF_BACKUP" ]; then
+ if [ -d "$LOGFEEDER_CONF_BACKUP/keys" ]; then
+ cp -r $LOGFEEDER_CONF_BACKUP/keys $LOGFEEDER_CONF_SOURCE
+ fi
+ if [ -d "$LOGFEEDER_CONF_BACKUP/checkpoints" ]; then
+ cp -r $LOGFEEDER_CONF_BACKUP/checkpoints $LOGFEEDER_CONF_SOURCE
+ fi
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh
new file mode 100755
index 0000000..5f1c623
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder"
+LOGFEEDER_CONF_DIR_LINK="/etc/ambari-logsearch-logfeeder/conf"
+
+rm -f $LOGFEEDER_SCRIPT_LINK_NAME
+rm -f $LOGFEEDER_CONF_DIR_LINK
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh
new file mode 100644
index 0000000..01d70e0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+LOGFEEDER_CONF="/etc/ambari-logsearch-logfeeder/conf"
+LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old"
+
+if [ -d $LOGFEEDER_CONF_BACKUP ]; then
+ rm -rf $LOGFEEDER_CONF_BACKUP
+fi
+
+if [ -d $LOGFEEDER_CONF ]; then
+ mkdir -p $LOGFEEDER_CONF_BACKUP
+ cp -r $LOGFEEDER_CONF/* $LOGFEEDER_CONF_BACKUP
+ rm -rf $LOGFEEDER_CONF
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh
new file mode 100644
index 0000000..9107482
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch"
+LOGSEARCH_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-portal/bin/logsearch.sh"
+
+LOGSEARCH_CONF_LINK="/etc/ambari-logsearch-portal/conf"
+LOGSEARCH_CONF_SOURCE="/usr/lib/ambari-logsearch-portal/conf"
+
+ln -s $LOGSEARCH_SCRIPT_SOURCE $LOGSEARCH_SCRIPT_LINK_NAME
+ln -s $LOGSEARCH_CONF_SOURCE $LOGSEARCH_CONF_LINK
+
+# handle old keys folder
+
+LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old"
+
+if [ -d "$LOGSEARCH_CONF_BACKUP" ]; then
+ if [ -d "$LOGSEARCH_CONF_BACKUP/keys" ]; then
+ cp -r $LOGSEARCH_CONF_BACKUP/keys $LOGSEARCH_CONF_SOURCE
+ fi
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh
new file mode 100644
index 0000000..8de05c1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch"
+LOGSEARCH_CONF_DIR_LINK="/etc/ambari-logsearch-portal/conf"
+
+rm -f $LOGSEARCH_SCRIPT_LINK_NAME
+rm -f $LOGSEARCH_CONF_DIR_LINK
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh
new file mode 100644
index 0000000..dfb8752
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+LOGSEARCH_CONF="/etc/ambari-logsearch-portal/conf"
+LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old"
+
+if [ -d $LOGSEARCH_CONF_BACKUP ]; then
+ rm -rf $LOGSEARCH_CONF_BACKUP
+fi
+
+if [ -d $LOGSEARCH_CONF ]; then
+ mkdir -p $LOGSEARCH_CONF_BACKUP
+ cp -r $LOGSEARCH_CONF/* $LOGSEARCH_CONF_BACKUP
+ rm -rf $LOGSEARCH_CONF
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/README.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
index d2e55f0..48cb81b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
@@ -29,11 +29,11 @@ First you need to start every required service (except logfeeder), go to `ambari
docker-compose up -d zookeeper solr logsearch
```
-Secondly, if you are planning to run Log Feeder from an IDE, for running the LogFeeder main methoud, you will need to set the working directory to `ambari/ambari-logsearch/ambari-logsearch-logfeeder` and use `--monitor` as a command line argument.
+Secondly, if you are planning to run Log Feeder from an IDE, for running the LogFeeder main methoud, you will need to set the working directory to `ambari/ambari-logsearch/ambari-logsearch-logfeeder` or set `LOGFEEDER_RELATIVE_LOCATION` env variable.
With Maven, you won't need these steps, just run this command from the ambari-logsearch-logfeeder folder:
```bash
-mvn clean package -DskipTests exec:java
+mvn clean package -DskipTests spring-boot:run
```
# Input Configuration
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/build.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/build.properties b/ambari-logsearch/ambari-logsearch-logfeeder/build.properties
deleted file mode 100644
index 013ba2e..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/build.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-app.pkg.dir=${app.work.dir}/pkg
-
-app.dev.war.dir=${app.work.dir}/webapps/logsearch
-app.war.name=logsearch.war
-
-app.target.dir=${builddir}/target/classes/static
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
index 738b2ef..c150b55 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml
@@ -19,10 +19,8 @@
<property environment="env"/>
<property name="debuglevel" value="source,lines,vars"/>
<dirname property="builddir" file="build.xml"/>
- <property name="target" value="1.7"/>
- <property name="source" value="1.7"/>
- <property file="local.properties"/>
- <property file="build.properties"/>
+ <property name="target" value="1.8"/>
+ <property name="source" value="1.8"/>
<target name="init">
</target>
<target name="build"/>
@@ -31,18 +29,20 @@
<delete dir="target/package"/>
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset dir="target/libs"/>
+ <fileset file="target/ambari-logsearch-logfeeder*.jar"/>
</copy>
- <copy todir="target/package/classes" includeEmptyDirs="no">
- <fileset dir="target/classes"/>
+ <copy todir="target/package/bin" includeEmptyDirs="no">
+ <fileset file="src/main/scripts/logfeeder.sh"/>
</copy>
- <copy todir="target/package" includeEmptyDirs="no">
- <fileset file="config.json"/>
+ <copy todir="target/package/conf" includeEmptyDirs="no">
+ <fileset file="target/classes/grok-patterns"/>
+ <fileset file="target/classes/log4j.xml"/>
+ <fileset file="target/classes/logfeeder.properties"/>
+ <fileset file="target/classes/alias_config.json"/>
+ <fileset file="src/main/scripts/logfeeder-env.sh"/>
</copy>
- <copy todir="target/package" includeEmptyDirs="no">
- <fileset file="grok-patterns"/>
- <fileset file="src/main/scripts/run.sh"/>
- </copy>
- <chmod file="target/package/run.sh" perm="755"/>
+ <chmod file="target/package/bin/logfeeder.sh" perm="755"/>
+ <chmod file="target/package/conf/logfeeder-env.sh" perm="755"/>
<tar compression="gzip" destfile="target/ambari-logsearch-logfeeder.tgz">
<tarfileset mode="755" dir="target/package">
<include name="*.sh"/>
@@ -52,6 +52,4 @@
</tarfileset>
</tar>
</target>
- <target description="Build all projects which reference this project. Useful to propagate changes."
- name="build-refprojects"/>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index 005af15..6a3524d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -198,7 +198,6 @@
</dependency>
</dependencies>
<build>
- <finalName>LogFeeder</finalName>
<pluginManagement>
<plugins>
<plugin>
@@ -222,40 +221,10 @@
<target>${jdk.version}</target>
</configuration>
</plugin>
-
- <!-- Exec main class plugin -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <mainClass>org.apache.ambari.logfeeder.LogFeeder</mainClass>
- <arguments>
- <argument>--monitor</argument>
- </arguments>
- </configuration>
- </plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
- <configuration>
- <classifier>exec</classifier>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>repackage</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<!-- copy-dependencies -->
<plugin>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/run.sh
index 70947ec..47a9e6c 100755
--- a/ambari-logsearch/ambari-logsearch-logfeeder/run.sh
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/run.sh
@@ -14,6 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-echo "LOG FEEDER"
-mvn clean compile
-mvn exec:java
+mvn clean spring-boot:run
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
index 2d31e5a..235109c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
@@ -18,54 +18,21 @@
*/
package org.apache.ambari.logfeeder;
-import com.google.gson.GsonBuilder;
-import org.apache.ambari.logfeeder.common.LogEntryParseTester;
-import org.apache.commons.io.FileUtils;
-import org.apache.log4j.LogManager;
import org.springframework.boot.Banner;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.system.ApplicationPidFileWriter;
-import java.io.File;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
@SpringBootApplication(
scanBasePackages = {"org.apache.ambari.logfeeder"}
)
public class LogFeeder {
public static void main(String[] args) {
- LogFeederCommandLine cli = new LogFeederCommandLine(args);
- if (cli.isTest()) {
- test(cli);
- }
- String pidFile = System.getenv("PID_FILE") == null ? "logfeeder.pid" : System.getenv("PID_FILE");
+ String pidFile = System.getenv("LOGFEEDER_PID_FILE") == null ? "logfeeder.pid" : System.getenv("LOGFEEDER_PID_FILE");
new SpringApplicationBuilder(LogFeeder.class)
.bannerMode(Banner.Mode.OFF)
.listeners(new ApplicationPidFileWriter(pidFile))
.run(args);
}
-
- private static void test(LogFeederCommandLine cli) {
- try {
- LogManager.shutdown();
- String testLogEntry = cli.getTestLogEntry();
- String testShipperConfig = FileUtils.readFileToString(new File(cli.getTestShipperConfig()), Charset.defaultCharset());
- List<String> testGlobalConfigs = new ArrayList<>();
- for (String testGlobalConfigFile : cli.getTestGlobalConfigs().split(",")) {
- testGlobalConfigs.add(FileUtils.readFileToString(new File(testGlobalConfigFile), Charset.defaultCharset()));
- }
- String testLogId = cli.getTestLogId();
- Map<String, Object> result = new LogEntryParseTester(testLogEntry, testShipperConfig, testGlobalConfigs, testLogId).parse();
- String parsedLogEntry = new GsonBuilder().setPrettyPrinting().create().toJson(result);
- System.out.println("The result of the parsing is:\n" + parsedLogEntry);
- } catch (Exception e) {
- System.out.println("Exception occurred, could not test if log entry is parseable");
- e.printStackTrace(System.out);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
index 61e7a1e..a6fcb2a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
@@ -19,22 +19,25 @@
package org.apache.ambari.logfeeder;
+import com.google.gson.GsonBuilder;
+import org.apache.ambari.logfeeder.common.LogEntryParseTester;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.log4j.LogManager;
+import java.io.File;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
public class LogFeederCommandLine {
-
- private static final Logger LOG = LoggerFactory.getLogger(LogFeederCommandLine.class);
private static final String TEST_COMMAND = "test";
private static final String TEST_LOG_ENTRY_OPTION = "test-log-entry";
@@ -42,11 +45,9 @@ public class LogFeederCommandLine {
private static final String TEST_GLOBAL_CONFIG_OPTION = "test-global-config";
private static final String TEST_LOG_ID_OPTION = "test-log-id";
- private static final String COMMAND_LINE_SYNTAX = "java org.apache.ambari.logfeeder.LogFeeder -(monitor|test) [args]";
-
- private CommandLine cli;
+ private static final String COMMAND_LINE_SYNTAX = "java org.apache.ambari.logfeeder.LogFeederCommandLine --test [args]";
- public LogFeederCommandLine(String[] args) {
+ public static void main(String[] args) {
Options options = new Options();
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setDescPadding(10);
@@ -95,7 +96,7 @@ public class LogFeederCommandLine {
try {
CommandLineParser cmdLineParser = new DefaultParser();
- cli = cmdLineParser.parse(options, args);
+ CommandLine cli = cmdLineParser.parse(options, args);
if (cli.hasOption('h')) {
helpFormatter.printHelp(COMMAND_LINE_SYNTAX, options);
@@ -105,11 +106,12 @@ public class LogFeederCommandLine {
if (cli.hasOption("t")) {
command = TEST_COMMAND;
validateRequiredOptions(cli, command, testLogEntryOption, testShipperConfOption);
- } else {
- LOG.info("Start application in monitor mode ");
}
+ test(cli);
} catch (Exception e) {
- LOG.info("Error parsing command line parameters: {}. LogFeeder will be started in monitoring mode.", e.getMessage());
+ e.printStackTrace();
+ helpFormatter.printHelp(COMMAND_LINE_SYNTAX, options);
+ System.exit(1);
}
}
@@ -125,24 +127,23 @@ public class LogFeederCommandLine {
String.format("The following options required for '%s' : %s", command, StringUtils.join(requiredOptions, ",")));
}
}
-
- public boolean isTest() {
- return cli != null && cli.hasOption('t');
- }
-
- public String getTestLogEntry() {
- return cli.getOptionValue("tle");
- }
-
- public String getTestShipperConfig() {
- return cli.getOptionValue("tsc");
- }
-
- public String getTestGlobalConfigs() {
- return cli.getOptionValue("tgc");
- }
-
- public String getTestLogId() {
- return cli.getOptionValue("tli");
+
+ private static void test(CommandLine cli) {
+ try {
+ LogManager.shutdown();
+ String testLogEntry = cli.getOptionValue("tle");
+ String testShipperConfig = FileUtils.readFileToString(new File(cli.getOptionValue("tsc")), Charset.defaultCharset());
+ List<String> testGlobalConfigs = new ArrayList<>();
+ for (String testGlobalConfigFile : cli.getOptionValue("tgc").split(",")) {
+ testGlobalConfigs.add(FileUtils.readFileToString(new File(testGlobalConfigFile), Charset.defaultCharset()));
+ }
+ String testLogId = cli.getOptionValue("tli");
+ Map<String, Object> result = new LogEntryParseTester(testLogEntry, testShipperConfig, testGlobalConfigs, testLogId).parse();
+ String parsedLogEntry = new GsonBuilder().setPrettyPrinting().create().toJson(result);
+ System.out.println("The result of the parsing is:\n" + parsedLogEntry);
+ } catch (Exception e) {
+ System.out.println("Exception occurred, could not test if log entry is parseable");
+ e.printStackTrace(System.out);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
index 35c0e6a..037f4a1 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
@@ -21,12 +21,12 @@ package org.apache.ambari.logfeeder.common;
import java.io.BufferedInputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.lang.reflect.Type;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -62,6 +62,7 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.google.gson.reflect.TypeToken;
+import org.springframework.core.io.ClassPathResource;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@@ -146,6 +147,10 @@ public class ConfigHandler implements InputConfigMonitor {
private void loadConfigsUsingClassLoader(String configFileName) throws Exception {
try (BufferedInputStream fis = (BufferedInputStream) this.getClass().getClassLoader().getResourceAsStream(configFileName)) {
+ ClassPathResource configFile = new ClassPathResource(configFileName);
+ if (!configFile.exists()) {
+ throw new FileNotFoundException(configFileName);
+ }
String configData = IOUtils.toString(fis, Charset.defaultCharset());
loadConfigs(configData);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
deleted file mode 100644
index 40cd855..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-Package: [[artifactId]]
-Version: [[package-version]]-[[package-release]]
-Section: [[deb.section]]
-Priority: [[deb.priority]]
-Depends: [[deb.dependency.list]]
-Architecture: [[deb.architecture]]
-Description: [[description]]
-Maintainer: [[deb.publisher]]
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
deleted file mode 100644
index 21a01fa..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
deleted file mode 100644
index 21a01fa..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
deleted file mode 100644
index 21a01fa..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
deleted file mode 100644
index 21a01fa..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh
new file mode 100755
index 0000000..37e9930
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Log Feeder extra options
+export LOGFEEDER_JAVA_OPTS=${LOGFEEDER_JAVA_OPTS:-""}
+
+# Log Feeder debug options
+# export LOGFEEDER_DEBUG=true
+# exoprt LOGFEEDER_DEBUG_SUSPEND=n
+export LOGFEEDER_DEBUG_PORT=5006
+
+# Log Feeder memory
+# export LOGFEEDER_JAVA_MEM="-Xmx512m"
+
+# export LOG_PATH=/var/log/ambari-logsearch-logfeeder/
+# export LOG_FILE=logfeeder.log
+
+# Pid file of the application
+# export LOGFEEDER_PID_DIR=/var/run/ambari-logsearch-logfeeder
+# export LOGFEEDER_PID_FILE=logfeeder.pid
+
+# SSL settings"
+# export LOGFEEDER_SSL="true"
+# export LOGFEEDER_KEYSTORE_LOCATION="/my/path/keystore.jks"
+# export LOGFEEDER_KEYSTORE_TYPE="jks"
+# export LOGFEEDER_TRUSTSTORE_LOCATION="/my/path/trutstore.jks"
+# export LOGFEEDER_TRUSTSTORE_TYPE="jks"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
new file mode 100755
index 0000000..cfacea5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
@@ -0,0 +1,292 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JVM="java"
+
+if [ -x $JAVA_HOME/bin/java ]; then
+ JVM=$JAVA_HOME/bin/java
+fi
+
+if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
+ LOGFEEDER_JAVA_MEM="-Xmx512m"
+fi
+
+readlinkf(){
+ # get real path on mac OSX
+ perl -MCwd -e 'print Cwd::abs_path shift' "$1";
+}
+
+if [ "$(uname -s)" = 'Linux' ]; then
+ SCRIPT_DIR="`dirname "$(readlink -f "$0")"`"
+else
+ SCRIPT_DIR="`dirname "$(readlinkf "$0")"`"
+fi
+
+LOGFEEDER_ROOT_DIR="`dirname \"$SCRIPT_DIR\"`"
+LOGFEEDER_LIBS_DIR="$LOGFEEDER_ROOT_DIR/libs"
+
+if [ "$LOGFEEDER_CONF_DIR" = "" ]; then
+ LOGFEEDER_CONF_DIR="/etc/ambari-logsearch-logfeeder/conf"
+ if [ ! -d $LOGFEEDER_CONF_DIR ]; then
+ if [ -d "$LOGFEEDER_ROOT_DIR/conf" ]; then
+ LOGFEEDER_CONF_DIR="$LOGFEEDER_ROOT_DIR/conf"
+ fi
+ fi
+fi
+
+if [ -f "$LOGFEEDER_CONF_DIR/logfeeder-env.sh" ]; then
+ source $LOGFEEDER_CONF_DIR/logfeeder-env.sh
+fi
+
+if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then
+ source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE
+fi
+
+if [ -z "$LOGFEEDER_PID_FILE" ]; then
+ LOGFEEDER_PID_DIR=$HOME
+ export LOGFEEDER_PID_FILE=$LOGFEEDER_PID_DIR/logfeeder.pid
+fi
+
+if [ -z "$LOG_FILE" ]; then
+ export LOG_FILE="logfeeder.log"
+fi
+
+LOGFEEDER_GC_LOGFILE="logfeeder_gc.log"
+
+if [ -z "$LOG_PATH" ]; then
+ LOG_FILE="$HOME/$LOG_FILE"
+ LOGFEEDER_GC_LOGFILE="$HOME/$LOGFEEDER_GC_LOGFILE"
+else
+ LOG_PATH_WITHOUT_SLASH=${LOG_PATH%/}
+ LOG_FILE="$LOG_PATH_WITHOUT_SLASH/$LOG_FILE"
+ LOGFEEDER_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGFEEDER_GC_LOGFILE"
+fi
+
+LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE"
+
+function print_usage() {
+ cat << EOF
+
+ Usage: [<command>] [<arguments with flags>]
+
+ commands:
+ start Start Log Feeder
+ stop Stop Log Feeder
+ status Check Log Feeder status (pid file)
+ test Test Log Feeder shipper configs
+ help Print usage
+
+
+ start command arguments:
+ -d, --debug Start java process in debug mode
+ -f, --foreground Start java process in foreground
+
+ test command arguments:
+ -h, --help Print usage
+ -tle, --test-log-entry Log entry to test if it's parseable (required)
+ -tsc, --test-shipper-config Shipper configuration file for testing if log entry is parseable (required)
+ -tgc, --test-global-config Global configuration files (comma separated list) for testing if log entry is parseable
+ -tli, --test-log-id The id of the log to test
+
+EOF
+}
+
+function spinner() {
+ local pid=$1
+ local delay=0.5
+ local spinstr='|/-\'
+ while [ "$(ps aux | awk '{print $2}' | grep -w $pid)" ]; do
+ local temp=${spinstr#?}
+ printf " [%c] " "$spinstr"
+ local spinstr=$temp${spinstr%"$temp"}
+ sleep $delay
+ printf "\b\b\b\b\b\b"
+ done
+ printf " \b\b\b\b"
+}
+
+function status() {
+ echo "Checking Log Feeder status ..."
+ if [ -f "$LOGFEEDER_PID_FILE" ]; then
+ LOGFEEDER_PID=`cat "$LOGFEEDER_PID_FILE"`
+ else
+ echo "Log Feeder pid not exists. (probably the process is not running)"
+ exit 1
+ fi
+
+ if ps -p $LOGFEEDER_PID > /dev/null
+ then
+ echo "Log Feeder process is running. (pid: $LOGFEEDER_PID)"
+ exit 0
+ else
+ echo "Log Feeder process is not running."
+ exit 1
+ fi
+}
+
+function start() {
+ (status);
+ exit_status=$(echo $?)
+ if [ "$exit_status" = "0" ]; then
+ echo "Skipping start process."
+ exit 0
+ fi
+
+ FG="false"
+ LOGFEEDER_DEBUG_SUSPEND=${LOGFEEDER_DEBUG_SUSPEND:-n}
+ LOGFEEDER_DEBUG_PORT=${LOGFEEDER_DEBUG_PORT:-"5006"}
+
+ if [ "$LOGFEEDER_DEBUG" = "true" ]; then
+ LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND "
+ fi
+
+ if [ "$LOGFEEDER_SSL" = "true" ]; then
+ LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGFEEDER_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGFEEDER_KEYSTORE_TYPE -Djavax.net.ssl.trustStore=$LOGFEEDER_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGFEEDER_TRUSTSTORE_TYPE"
+ fi
+
+ if [ "$LOGFEEDER_JMX" = "true" ]; then
+ LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098"
+ fi
+
+ if [ $# -gt 0 ]; then
+ while true; do
+ case "$1" in
+ -f|--foreground)
+ FG="true"
+ shift
+ ;;
+ -d|--debug)
+ if [ "$LOGFEEDER_DEBUG" != "true" ]; then
+ LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND "
+ fi
+ shift
+ ;;
+ *)
+ if [ "${1:0:2}" == "-D" ]; then
+ # pass thru any opts that begin with -D (java system props)
+ LOGFEEDER_JAVA_OPTS+=("$1")
+ echo "$LOGFEEDER_JAVA_OPTS"
+ shift
+ else
+ if [ "$1" != "" ]; then
+ print_usage
+ exit 1
+ else
+ break
+ fi
+ fi
+ ;;
+ esac
+ done
+ fi
+
+ if [ $FG == "true" ]; then
+ echo "Starting logfeeder... (foreground) pid_file=$LOGFEEDER_PID_FILE"
+ echo "Run command $JVM -cp '$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*' $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder"
+ $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder
+ else
+ echo "Starting logfeeder... Output file=$LOG_FILE pid_file=$LOGFEEDER_PID_FILE"
+ echo "Run command nohup $JVM -cp '$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*' $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder"
+ nohup $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder > $LOG_FILE 2>&1 &
+ fi
+}
+
+function stop() {
+ LOGFEEDER_STOP_WAIT=3
+ if [ -f "$LOGFEEDER_PID_FILE" ]; then
+ LOGFEEDER_PID=`cat "$LOGFEEDER_PID_FILE"`
+ fi
+
+ if [ "$LOGFEEDER_PID" != "" ]; then
+ echo -e "Sending stop command to Log Feeder... Checking PID: $LOGFEEDER_PID."
+ kill $LOGFEEDER_PID
+ (loops=0
+ while true
+ do
+ CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGFEEDER_PID | sort -r | tr -d ' '`
+ if [ "$CHECK_PID" != "" ]; then
+ slept=$((loops * 2))
+ if [ $slept -lt $LOGFEEDER_STOP_WAIT ]; then
+ sleep 2
+ loops=$[$loops+1]
+ else
+ exit # subshell!
+ fi
+ else
+ exit # subshell!
+ fi
+ done) &
+ spinner $!
+ rm -f "$LOGFEEDER_PID_FILE"
+ else
+ echo -e "No LogFeeder process found to stop."
+ exit 0
+ fi
+
+ CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGFEEDER_PID | sort -r | tr -d ' '`
+ if [ "$CHECK_PID" != "" ]; then
+ echo -e "Log Feeder process $LOGFEEDER_PID is still running; forcefully killing it now."
+ kill -9 $LOGFEEDER_PID
+ echo "Killed process $LOGFEEDER_PID"
+ rm -f "$LOGFEEDER_PID_FILE"
+ sleep 1
+ else
+ echo "Log Feeder is stopped."
+ fi
+
+ CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGFEEDER_PID | sort -r | tr -d ' '`
+ if [ "$CHECK_PID" != "" ]; then
+ echo "ERROR: Failed to kill Log Feeder Java process $LOGFEEDER_PID ... script fails."
+ exit 1
+ fi
+}
+
+function test() {
+ echo "Running command: $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" org.apache.ambari.logfeeder.LogFeederCommandLine --test ${@}"
+ $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" $LOGFEEDER_JAVA_OPTS org.apache.ambari.logfeeder.LogFeederCommandLine --test ${@}
+}
+
+if [ $# -gt 0 ]; then
+ SCRIPT_CMD="$1"
+ shift
+else
+ print_usage
+ exit 1
+fi
+
+case $SCRIPT_CMD in
+ start)
+ start ${1+"$@"}
+ ;;
+ stop)
+ stop
+ ;;
+ status)
+ status
+ ;;
+ test)
+ test ${1+"$@"}
+ ;;
+ help)
+ print_usage
+ exit 0
+ ;;
+ *)
+ print_usage
+ exit 1
+ ;;
+
+esac
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
deleted file mode 100644
index 22e1cb2..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-curr_dir=`pwd`
-cd `dirname $0`; script_dir=`pwd`; cd $curr_dir
-
-foreground=0
-if [ "$1" = "-foreground" ]; then
- foreground=1
- shift
-fi
-
-if [ ! -z "$LOGFEEDER_INCLUDE" ]; then
- source $LOGFEEDER_INCLUDE
-fi
-
-if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then
- source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE
-fi
-
-JAVA=java
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
-fi
-
-if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
- LOGFEEDER_JAVA_MEM="-Xmx512m"
-fi
-
-if [ "$LOGFILE" = "" ]; then
- LOGFILE="/var/log/logfeeder/logfeeder.out"
-fi
-
-if [ "$PID_FILE" = "" ]; then
- LOGFEEDER_PID_DIR=$HOME
- PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid
-fi
-
-if [ "$LOGFEEDER_CONF_DIR" = "" ]; then
- LOGFEEDER_CONF_DIR="/etc/logfeeder/conf"
- if [ ! -d $LOGFEEDER_CONF_DIR ]; then
- if [ -d $script_dir/classes ]; then
- LOGFEEDER_CONF_DIR=$script_dir/classes
- fi
- fi
-fi
-
-LOGFEEDER_DEBUG_SUSPEND=${LOGFEEDER_DEBUG_SUSPEND:-n}
-if [ "$LOGFEEDER_DEBUG" = "true" ] && [ ! -z "$LOGFEEDER_DEBUG_PORT" ]; then
- LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND "
-fi
-
-LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log
-LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE"
-
-#JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098"
-
-if [ "$LOGFEEDER_SSL" = "true" ]; then
- LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGFEEDER_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGFEEDER_KEYSTORE_TYPE -Djavax.net.ssl.trustStore=$LOGFEEDER_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGFEEDER_TRUSTSTORE_TYPE"
-fi
-
-if [ $foreground -eq 0 ]; then
- if [ -f ${PID_FILE} ]; then
- PID=`cat ${PID_FILE}`
- if kill -0 $PID 2>/dev/null; then
- echo "logfeeder already running (${PID}) killing..."
- kill $PID 2>/dev/null
- sleep 5
- if kill -0 $PID 2>/dev/null; then
- echo "logfeeder still running. Will kill process forcefully in another 10 seconds..."
- sleep 10
- kill -9 $PID 2>/dev/null
- sleep 2
- fi
- fi
-
- if kill -0 $PID 2>/dev/null; then
- echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again."
- exit 1
- fi
- fi
-
- echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE"
- #LOGFEEDER_CLI_CLASSPATH=set -x
-nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder --monitor $* > $LOGFILE 2>&1 &
- echo $! > $PID_FILE
-else
-$JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder --monitor $*
-fi
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/README.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/README.md b/ambari-logsearch/ambari-logsearch-server/README.md
index 26b1f73..7432344 100644
--- a/ambari-logsearch/ambari-logsearch-server/README.md
+++ b/ambari-logsearch/ambari-logsearch-server/README.md
@@ -33,7 +33,7 @@ Then you can start Log Search server from maven
cd ambari/ambari-logsearch/ambari-logsearch-server
./run.sh
# or
-mvn clean spring-boot:run
+mvn clean package -DskipTests spring-boot:run
```
You can also start Log Search server from an IDE as well. One thing is important: the config set location that the server tries to upload to ZooKeeper. By default config sets are located at `${LOGSEARCH_SERVER_RELATIVE_LOCATION:}src/main/configsets` in `logsearch.properties`. Based or from where you run `LogSearch.java`, you need to set `LOGSEARCH_SERVER_RELATIVE_LOCATION` env variable properly.
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/build.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/build.xml b/ambari-logsearch/ambari-logsearch-server/build.xml
index 5c50485..c711fe1 100644
--- a/ambari-logsearch/ambari-logsearch-server/build.xml
+++ b/ambari-logsearch/ambari-logsearch-server/build.xml
@@ -19,9 +19,8 @@
<property environment="env"/>
<property name="debuglevel" value="source,lines,vars"/>
<dirname property="builddir" file="build.xml"/>
- <property name="target" value="1.7"/>
- <property name="source" value="1.7"/>
- <property file="build.properties"/>
+ <property name="target" value="1.8"/>
+ <property name="source" value="1.8"/>
<target name="init">
</target>
@@ -31,20 +30,34 @@
<delete dir="target/package"/>
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset dir="target/libs"/>
+ <fileset file="target/ambari-logsearch-server*.jar"/>
</copy>
<copy todir="target/classes/static" includeEmptyDirs="no">
<fileset dir="target/dist"/>
</copy>
- <copy todir="target/package/classes" includeEmptyDirs="no">
- <fileset dir="target/classes"/>
+ <copy todir="target/package/webapp/static" includeEmptyDirs="no">
+ <fileset dir="target/classes/static"/>
</copy>
- <copy todir="target/package/solr_configsets" includeEmptyDirs="yes">
+ <copy todir="target/package/webapp/swagger" includeEmptyDirs="no">
+ <fileset dir="target/classes/swagger"/>
+ </copy>
+ <copy todir="target/package/conf" includeEmptyDirs="no">
+ <fileset file="src/main/scripts/logsearch-env.sh"/>
+ <fileset file="target/classes/log4j.xml"/>
+ <fileset file="target/classes/default.properties"/>
+ <fileset file="target/classes/logsearch.properties"/>
+ <fileset file="target/classes/info.properties"/>
+ <fileset file="target/classes/user_pass.json"/>
+ <fileset file="target/classes/HadoopServiceConfig.json"/>
+ </copy>
+ <copy todir="target/package/conf/solr_configsets" includeEmptyDirs="yes">
<fileset dir="src/main/configsets"/>
</copy>
- <copy todir="target/package" includeEmptyDirs="no">
- <fileset file="src/main/scripts/*"/>
+ <copy todir="target/package/bin" includeEmptyDirs="no">
+ <fileset file="src/main/scripts/logsearch.sh"/>
</copy>
- <chmod file="target/package/*.sh" perm="755"/>
+ <chmod file="target/package/bin/logsearch.sh" perm="755"/>
+ <chmod file="target/package/conf/logsearch-env.sh" perm="755"/>
<tar compression="gzip" destfile="target/ambari-logsearch-portal.tar.gz">
<tarfileset mode="755" dir="target/package">
<include name="*.sh"/>
@@ -53,10 +66,5 @@
<exclude name="*.sh"/>
</tarfileset>
</tar>
-
</target>
-
-
- <target description="Build all projects which reference this project. Useful to propagate changes."
- name="build-refprojects"/>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/pom.xml b/ambari-logsearch/ambari-logsearch-server/pom.xml
index 5444b00..6a7debf 100755
--- a/ambari-logsearch/ambari-logsearch-server/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-server/pom.xml
@@ -44,7 +44,6 @@
<activeByDefault>true</activeByDefault>
</activation>
<build>
- <finalName>LogSearch</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
@@ -77,16 +76,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
- <configuration>
- <classifier>exec</classifier>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>repackage</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/run.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/run.sh b/ambari-logsearch/ambari-logsearch-server/run.sh
index f245930..2de0536 100755
--- a/ambari-logsearch/ambari-logsearch-server/run.sh
+++ b/ambari-logsearch/ambari-logsearch-server/run.sh
@@ -17,4 +17,4 @@ echo "
███████╗╚██████╔╝╚██████╔╝ ███████║███████╗██║ ██║██║ ██║╚██████╗██║ ██║
╚══════╝ ╚═════╝ ╚═════╝ ╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝
"
-mvn clean spring-boot:run
+mvn clean package -DskipTests spring-boot:run
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java
index 39343c7..45cda0a 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java
@@ -36,7 +36,7 @@ public class LogSearch {
public static void main(String[] args) {
- String pidFile = System.getenv("PID_FILE") == null ? "logsearch.pid" : System.getenv("PID_FILE");
+ String pidFile = System.getenv("LOGSEARCH_PID_FILE") == null ? "logsearch.pid" : System.getenv("LOGSEARCH_PID_FILE");
new SpringApplicationBuilder(LogSearch.class)
.bannerMode(Banner.Mode.OFF)
.listeners(new ApplicationPidFileWriter(pidFile))
http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh
new file mode 100755
index 0000000..d8a837f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Log Search extra options
+export LOGSEARCH_JAVA_OPTS=${LOGSEARCH_JAVA_OPTS:-""}
+
+# Log Search debug options
+# export LOGSEARCH_DEBUG=true
+# exoprt LOGSEARCH_DEBUG_SUSPEND=n
+export LOGSEARCH_DEBUG_PORT=5005
+
+# Log Search memory
+# export LOGSEARCH_JAVA_MEM="--Xmx1024m"
+
+# export LOG_PATH=/var/log/ambari-logsearch-logfeeder/
+# export LOG_FILE=logsearch.log
+
+# Pid file of the application
+# export LOGSEARCH_PID_DIR=/var/run/ambari-logsearch-logfeeder
+# export LOGSEARCH_PID_FILE=logfeeder.pid
+
+# SSL settings"
+# export LOGSEARCH_SSL="true"
+# export LOGSEARCH_KEYSTORE_LOCATION="/my/path/keystore.jks"
+# export LOGSEARCH_KEYSTORE_TYPE="jks"
+# export LOGSEARCH_TRUSTSTORE_LOCATION="/my/path/trutstore.jks"
+# export LOGSEARCH_TRUSTSTORE_TYPE="jks"
\ No newline at end of file
[22/51] [abbrv] ambari git commit: AMBARI-22672. Error on add version
view in web admin (alexantonenko)
Posted by am...@apache.org.
AMBARI-22672. Error on add version view in web admin (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bb0980d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bb0980d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bb0980d
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 9bb0980da1261f3be769ba26db9adb5ebaa74a55
Parents: 30d89a3
Author: Alex Antonenko <aa...@hortonworks.com>
Authored: Tue Dec 19 16:42:37 2017 +0300
Committer: Alex Antonenko <aa...@hortonworks.com>
Committed: Tue Dec 19 16:42:37 2017 +0300
----------------------------------------------------------------------
.../scripts/controllers/stackVersions/StackVersionsCreateCtrl.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb0980d/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js
index 3d93b0d..2a2151d 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js
@@ -43,7 +43,7 @@ angular.module('ambariAdminConsole')
$scope.isGPLAccepted = false;
$scope.isGPLRepo = function (repository) {
- return repository.Repositories.tags.indexOf('GPL') >= 0;
+ return repository.Repositories.tags && repository.Repositories.tags.indexOf('GPL') >= 0;
};
$scope.showRepo = function (repository) {
[13/51] [abbrv] ambari git commit: AMBARI-22640 - HBase Cannot Find
LZO Classes After Being Patched (jonathanhurley)
Posted by am...@apache.org.
AMBARI-22640 - HBase Cannot Find LZO Classes After Being Patched (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eab6722f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eab6722f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eab6722f
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: eab6722f705f24a1d731fa39b5e220578e231478
Parents: 562795c
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Dec 12 16:18:47 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Dec 18 14:34:52 2017 -0500
----------------------------------------------------------------------
.../HBASE/0.96.0.2.0/package/scripts/hbase.py | 26 ++++++++-----
.../HBASE/2.0.0.3.0/package/scripts/hbase.py | 39 ++++++++++++--------
2 files changed, 41 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/eab6722f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
index cec6b2a..6490b59 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
@@ -18,18 +18,23 @@ limitations under the License.
"""
from urlparse import urlparse
+
import os
-import sys
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.resources.template_config import TemplateConfig
-from resource_management.libraries.functions.format import format
-from resource_management.core.source import Template, InlineTemplate
-from resource_management.core.resources import Package
-from resource_management.core.resources.service import ServiceConfig
-from resource_management.core.resources.system import Directory, Execute, File
+
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.core.resources import Directory
+from resource_management.core.resources import Execute
+from resource_management.core.resources import File
+from resource_management.core.resources import Package
+from resource_management.core.resources import ServiceConfig
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.libraries import Script
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import lzo_utils
+from resource_management.libraries.resources import TemplateConfig
+from resource_management.libraries.resources import XmlConfig
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.stack_features import check_stack_feature
@@ -55,6 +60,9 @@ def hbase(name=None):
def hbase(name=None):
import params
+ # ensure that matching LZO libraries are installed for HBase
+ lzo_utils.install_lzo_if_needed()
+
Directory( params.etc_prefix_dir,
mode=0755
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/eab6722f/ambari-server/src/main/resources/common-services/HBASE/2.0.0.3.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/2.0.0.3.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/2.0.0.3.0/package/scripts/hbase.py
index fced4fc..1a25095 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/2.0.0.3.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/2.0.0.3.0/package/scripts/hbase.py
@@ -17,18 +17,24 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+from urlparse import urlparse
+
import os
-import sys
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.resources.template_config import TemplateConfig
-from resource_management.libraries.functions.format import format
-from resource_management.core.source import Template, InlineTemplate
-from resource_management.core.resources import Package
-from resource_management.core.resources.service import ServiceConfig
-from resource_management.core.resources.system import Directory, Execute, File
+
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.core.resources import Directory
+from resource_management.core.resources import Execute
+from resource_management.core.resources import File
+from resource_management.core.resources import Package
+from resource_management.core.resources import ServiceConfig
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.libraries import Script
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import lzo_utils
+from resource_management.libraries.resources import TemplateConfig
+from resource_management.libraries.resources import XmlConfig
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.stack_features import check_stack_feature
@@ -54,6 +60,9 @@ def hbase(name=None):
def hbase(name=None):
import params
+ # ensure that matching LZO libraries are installed for HBase
+ lzo_utils.install_lzo_if_needed()
+
Directory( params.etc_prefix_dir,
mode=0755
)
@@ -63,7 +72,7 @@ def hbase(name=None):
group = params.user_group,
create_parents = True
)
-
+
Directory(params.java_io_tmpdir,
create_parents = True,
mode=0777
@@ -83,7 +92,7 @@ def hbase(name=None):
create_parents = True,
mode = 0755
)
-
+
parent_dir = os.path.dirname(params.tmp_dir)
# In case if we have several placeholders in path
while ("${" in parent_dir):
@@ -136,7 +145,7 @@ def hbase(name=None):
group = params.user_group
)
# Manually overriding ownership of file installed by hadoop package
- else:
+ else:
File( format("{params.hbase_conf_dir}/hbase-policy.xml"),
owner = params.hbase_user,
group = params.user_group
@@ -147,14 +156,14 @@ def hbase(name=None):
content=InlineTemplate(params.hbase_env_sh_template),
group = params.user_group,
)
-
+
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
-
+
File(os.path.join(params.limits_conf_dir, 'hbase.conf'),
owner='root',
group='root',
@@ -174,7 +183,7 @@ def hbase(name=None):
cd_access = "a",
mode = 0755,
)
-
+
Directory (params.log_dir,
owner = params.hbase_user,
create_parents = True,
[46/51] [abbrv] ambari git commit: AMBARI-22424. Quicklinks for OneFS
(amagyar)
Posted by am...@apache.org.
AMBARI-22424. Quicklinks for OneFS (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a98d348
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a98d348
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a98d348
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 6a98d348115c6ffc91aeed2795fdfd48422de761
Parents: 95c46db
Author: Attila Magyar <am...@hortonworks.com>
Authored: Mon Nov 13 16:00:50 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../ambari/server/state/quicklinks/Host.java | 75 ++++++++++++++++++++
.../ambari/server/state/quicklinks/Link.java | 13 ++++
.../QuickLinksConfigurationModuleTest.java | 3 +
.../resources/child_quicklinks_to_override.json | 4 ++
.../app/views/common/quick_view_link_view.js | 64 +++++++++++++----
.../test/views/common/quick_link_view_test.js | 15 +++-
.../addon-services/ONEFS/1.0.0/metainfo.xml | 7 ++
.../ONEFS/1.0.0/quicklinks/quicklinks.json | 31 ++++++++
8 files changed, 197 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Host.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Host.java
new file mode 100644
index 0000000..074d4e7
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Host.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state.quicklinks;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Quick links may override host names of host components with host names that come from configuration.
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Host {
+ /**
+ * The property name that has the host name if the protocol is http
+ */
+ @JsonProperty("http_property")
+ private String httpProperty;
+
+ /**
+ * The property name that has the host name if the protocol is https
+ */
+ @JsonProperty("https_property")
+ private String httpsProperty;
+
+ /**
+ * The config type where the overridden host name comes from
+ */
+ @JsonProperty("site")
+ private String site;
+
+ public String getHttpProperty() {
+ return httpProperty;
+ }
+
+ public String getHttpsProperty() {
+ return httpsProperty;
+ }
+
+ public String getSite() {
+ return site;
+ }
+
+ public void mergeWithParent(Host parentHost) {
+ if(null == parentHost) {
+ return;
+ }
+ if(null == httpProperty && null != parentHost.getHttpProperty()) {
+ httpProperty = parentHost.getHttpProperty();
+ }
+ if(null == httpsProperty && null != parentHost.getHttpsProperty()) {
+ httpsProperty = parentHost.getHttpsProperty();
+ }
+ if(null == site && null != parentHost.getSite()) {
+ site = parentHost.getSite();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Link.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Link.java b/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Link.java
index 1d2e712..4e8e8a3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Link.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/quicklinks/Link.java
@@ -47,6 +47,9 @@ public class Link{
@JsonProperty("port")
private Port port;
+ @JsonProperty("host")
+ private Host host;
+
@JsonProperty("protocol")
private Protocol protocol;
@@ -100,6 +103,10 @@ public class Link{
return port;
}
+ public Host getHost() {
+ return host;
+ }
+
public void setPort(Port port) {
this.port = port;
}
@@ -159,6 +166,12 @@ public class Link{
port.mergetWithParent(parentLink.getPort());
}
+ if(null == host){
+ host = parentLink.getHost();
+ } else {
+ host.mergeWithParent(parentLink.getHost());
+ }
+
if (null == attributes && null != parentLink.attributes) {
attributes = parentLink.attributes;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/ambari-server/src/test/java/org/apache/ambari/server/stack/QuickLinksConfigurationModuleTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/QuickLinksConfigurationModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/QuickLinksConfigurationModuleTest.java
index aac278d..040d630 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/QuickLinksConfigurationModuleTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/QuickLinksConfigurationModuleTest.java
@@ -33,6 +33,7 @@ import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.state.quicklinks.Check;
+import org.apache.ambari.server.state.quicklinks.Host;
import org.apache.ambari.server.state.quicklinks.Link;
import org.apache.ambari.server.state.quicklinks.Port;
import org.apache.ambari.server.state.quicklinks.Protocol;
@@ -127,6 +128,8 @@ public class QuickLinksConfigurationModuleTest {
hasLink = true;
Port port = link.getPort();
assertEquals("mapred-site", port.getSite());
+ Host host = link.getHost();
+ assertEquals("core-site", host.getSite());
}
}
assertTrue(hasLink);
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/ambari-server/src/test/resources/child_quicklinks_to_override.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/child_quicklinks_to_override.json b/ambari-server/src/test/resources/child_quicklinks_to_override.json
index 4309532..0aaec1f 100644
--- a/ambari-server/src/test/resources/child_quicklinks_to_override.json
+++ b/ambari-server/src/test/resources/child_quicklinks_to_override.json
@@ -79,6 +79,10 @@
"https_default_port": "8090",
"regex": "\\w*:(\\d+)",
"site": "mapred-site"
+ },
+ "host":{
+ "http_property": "fs.defaultFS",
+ "site": "core-site"
}
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/ambari-web/app/views/common/quick_view_link_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/quick_view_link_view.js b/ambari-web/app/views/common/quick_view_link_view.js
index 5888acb..3ccc973 100644
--- a/ambari-web/app/views/common/quick_view_link_view.js
+++ b/ambari-web/app/views/common/quick_view_link_view.js
@@ -206,11 +206,8 @@ App.QuickLinksView = Em.View.extend({
if (!Em.isEmpty(links)) {
links.forEach(function (link) {
if (!link.remove) {
- var portConfig = Em.get(link, 'port');
- var portConfigSiteProp = Em.get(portConfig, 'site');
- if (!sites.contains(portConfigSiteProp)) {
- sites.push(portConfigSiteProp);
- }
+ this.addSite(link, 'host', sites);
+ this.addSite(link, 'port', sites);
}
}, this);
this.set('requiredSiteNames', this.get('requiredSiteNames').pushObjects(sites).uniq());
@@ -222,6 +219,16 @@ App.QuickLinksView = Em.View.extend({
}
},
+ addSite: function(link, linkPropertyName, sites) {
+ var config = Em.get(link, linkPropertyName);
+ if (config) {
+ var siteName = Em.get(config, 'site');
+ if (!sites.contains(siteName)) {
+ sites.push(siteName);
+ }
+ }
+ },
+
/**
* call for public host names
*
@@ -267,8 +274,8 @@ App.QuickLinksView = Em.View.extend({
});
// no need to set quicklinks if
// 1)current service does not have quick links configured
- // 2)No host component present for the configured quicklinks
- if(hasQuickLinks && hasHosts) {
+ // 2)No host component present for the configured quicklinks and has no overridden hosts
+ if(hasQuickLinks && (hasHosts || this.hasOverriddenHost())) {
this.set('showQuickLinks', true);
} else {
this.set('showNoLinks', true);
@@ -276,15 +283,20 @@ App.QuickLinksView = Em.View.extend({
var isMultipleComponentsInLinks = componentNames.uniq().length > 1;
- if (hosts.length === 0) {
+ if (hosts.length === 0 && !this.hasOverriddenHost()) {
this.setEmptyLinks();
- } else if (hosts.length === 1 || isMultipleComponentsInLinks) {
+ } else if (hosts.length === 1 || isMultipleComponentsInLinks || this.hasOverriddenHost()) {
this.setSingleHostLinks(hosts, response);
} else {
this.setMultipleHostLinks(hosts);
}
},
+ hasOverriddenHost: function() {
+ var links = Em.get(this.getQuickLinksConfiguration(), 'links');
+ return links && links.some(function (each) { return each.host; });
+ },
+
/**
* Get public host name by its host name.
*
@@ -424,10 +436,8 @@ App.QuickLinksView = Em.View.extend({
var links = Em.get(quickLinksConfig, 'links');
links.forEach(function (link) {
- var componentName = link.component_name;
- var hostNameForComponent = hosts.findProperty('componentName',componentName);
- if (hostNameForComponent) {
- var publicHostName = hostNameForComponent.publicHostName;
+ var publicHostName = this.publicHostName(link, hosts, protocol);
+ if (publicHostName) {
if (link.protocol) {
protocol = this.setProtocol(configProperties, link.protocol);
}
@@ -446,6 +456,30 @@ App.QuickLinksView = Em.View.extend({
}
},
+ publicHostName: function(link, hosts, protocol) {
+ if (link.host) { // if quicklink overrides hostcomponent host name, get host from config
+ var configProperties = this.get('configProperties');
+ var hostProperty = Em.get(link.host, protocol + '_property');
+ var site = configProperties.findProperty('type', Em.get(link.host, 'site'));
+ return site && site.properties ? this.parseHostFromUri(site.properties[hostProperty]) : null;
+ } else {
+ var hostNameForComponent = hosts.findProperty('componentName', link.component_name);
+ return hostNameForComponent ? hostNameForComponent.publicHostName : null;
+ }
+ },
+
+ /**
+ * @param {string} uri
+ */
+ parseHostFromUri: function(uri) {
+ if (uri) {
+ var match = uri.match(/:\/\/([^/:]+)/i);
+ return match != null && match.length == 2 ? match[1] : uri;
+ } else {
+ return null;
+ }
+ },
+
/**
* set links that contain multiple hosts
*
@@ -786,7 +820,9 @@ App.QuickLinksView = Em.View.extend({
* @method setPort
*/
setPort: function (portConfigs, protocol, configProperties) {
-
+ if (!portConfigs) {
+ return '';
+ }
var defaultPort = Em.get(portConfigs, protocol + '_default_port');
var portProperty = Em.get(portConfigs, protocol + '_property');
var site = configProperties.findProperty('type', Em.get(portConfigs, 'site'));
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/ambari-web/test/views/common/quick_link_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/common/quick_link_view_test.js b/ambari-web/test/views/common/quick_link_view_test.js
index 1998a9c..9adbbc8 100644
--- a/ambari-web/test/views/common/quick_link_view_test.js
+++ b/ambari-web/test/views/common/quick_link_view_test.js
@@ -195,6 +195,9 @@ describe('App.QuickViewLinks', function () {
{
port: {
site: "yarn-site"
+ },
+ host: {
+ site: "yarn-env"
}
}
]
@@ -205,7 +208,7 @@ describe('App.QuickViewLinks', function () {
quickViewLinks.set('content.serviceName', 'YARN');
mock.returns(quickLinksConfigYARN);
quickViewLinks.loadQuickLinksConfigSuccessCallback({items: []});
- expect(quickViewLinks.get('requiredSiteNames')).to.be.eql(["core-site", "hdfs-site", "admin-properties", "hbase-site", "yarn-site"]);
+ expect(quickViewLinks.get('requiredSiteNames')).to.be.eql(["core-site", "hdfs-site", "admin-properties", "hbase-site", "yarn-site", "yarn-env"]);
});
});
@@ -247,8 +250,11 @@ describe('App.QuickViewLinks', function () {
});
describe("#setQuickLinksSuccessCallback()", function () {
+ var getQuickLinks;
beforeEach(function () {
this.mock = sinon.stub(quickViewLinks, 'getHosts');
+ getQuickLinks = sinon.stub(quickViewLinks, 'getQuickLinksConfiguration');
+ getQuickLinks.returns({});
sinon.stub(quickViewLinks, 'setEmptyLinks');
sinon.stub(quickViewLinks, 'setSingleHostLinks');
sinon.stub(quickViewLinks, 'setMultipleHostLinks');
@@ -256,6 +262,7 @@ describe('App.QuickViewLinks', function () {
});
afterEach(function () {
this.mock.restore();
+ getQuickLinks.restore();
quickViewLinks.setEmptyLinks.restore();
quickViewLinks.setSingleHostLinks.restore();
quickViewLinks.setMultipleHostLinks.restore();
@@ -265,6 +272,12 @@ describe('App.QuickViewLinks', function () {
quickViewLinks.setQuickLinksSuccessCallback();
expect(quickViewLinks.setEmptyLinks.calledOnce).to.be.true;
});
+ it("has overridden hosts", function () {
+ this.mock.returns([]);
+ getQuickLinks.returns({ links: [{ host: {site: "yarn-env"} }] });
+ quickViewLinks.setQuickLinksSuccessCallback();
+ expect(quickViewLinks.setEmptyLinks.calledOnce).to.be.false;
+ });
it("quickLinks is not configured", function () {
this.mock.returns([{}]);
quickViewLinks.setQuickLinksSuccessCallback();
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
index 82e2022..5ba562e 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
@@ -86,6 +86,13 @@
<config-type>hadoop-env</config-type>
</configuration-dependencies>
<restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
</service>
</services>
</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a98d348/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/quicklinks/quicklinks.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/quicklinks/quicklinks.json
new file mode 100644
index 0000000..0b9656f
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/quicklinks/quicklinks.json
@@ -0,0 +1,31 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol": {
+ "type": "HTTP_ONLY"
+ },
+ "links": [
+ {
+ "component_name" : "ONEFS_CLIENT",
+ "name": "onefs_web_ui",
+ "label": "OneFS Web UI",
+ "url": "%@://%@",
+ "host": {
+ "http_property": "fs.defaultFS",
+ "site": "core-site"
+ }
+ },
+ {
+ "component_name" : "ONEFS_CLIENT",
+ "name": "onefs_hdfs_web_ui",
+ "label": "OneFS HDFS Settings",
+ "url": "%@://%@/OneFS#HDFS/Settings",
+ "host": {
+ "http_property": "fs.defaultFS",
+ "site": "core-site"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
[37/51] [abbrv] ambari git commit: AMBARI-22514,
AMBARI-22653. Ambari Infra Manager: solr data exporting jobs and
integration test environment. (Krisztian Kasa via swagle)
Posted by am...@apache.org.
AMBARI-22514, AMBARI-22653. Ambari Infra Manager: solr data exporting jobs and integration test environment. (Krisztian Kasa via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36d0271f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36d0271f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36d0271f
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 36d0271f74a70f5cfeca0e5ca0ebeb795fab6138
Parents: a15fc7f
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Dec 21 13:24:03 2017 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Dec 21 13:24:03 2017 -0800
----------------------------------------------------------------------
ambari-infra/ambari-infra-manager-it/pom.xml | 155 +++++++++++++
.../org/apache/ambari/infra/InfraClient.java | 93 ++++++++
.../ambari/infra/InfraManagerStories.java | 108 +++++++++
.../ambari/infra/OffsetDateTimeConverter.java | 39 ++++
.../ambari/infra/steps/AbstractInfraSteps.java | 223 +++++++++++++++++++
.../ambari/infra/steps/ExportJobsSteps.java | 106 +++++++++
.../src/test/resources/log4j.properties | 16 ++
.../resources/stories/infra_api_tests.story | 23 ++
.../ambari-infra-manager/docker/Dockerfile | 6 +-
.../docker/docker-compose.yml | 81 +++++++
.../docker/infra-manager-docker-compose.sh | 105 +++++++++
.../apache/ambari/infra/job/ObjectSource.java | 23 ++
.../infra/job/archive/AbstractFileAction.java | 33 +++
.../infra/job/archive/CompositeFileAction.java | 7 +-
.../ambari/infra/job/archive/Document.java | 1 -
.../archive/DocumentExportConfiguration.java | 74 +++---
.../job/archive/DocumentExportJobListener.java | 23 ++
.../job/archive/DocumentExportProperties.java | 140 +++++++++---
.../job/archive/DocumentExportPropertyMap.java | 38 ++++
.../job/archive/DocumentExportStepListener.java | 47 ----
.../infra/job/archive/DocumentItemReader.java | 8 +-
.../infra/job/archive/DocumentIterator.java | 5 +-
.../infra/job/archive/DocumentSource.java | 7 +-
.../ambari/infra/job/archive/FileAction.java | 2 +-
.../job/archive/LocalDocumentItemWriter.java | 8 +-
.../ambari/infra/job/archive/S3Properties.java | 57 ++---
.../ambari/infra/job/archive/S3Uploader.java | 23 +-
.../infra/job/archive/SolrDocumentIterator.java | 3 +-
.../infra/job/archive/SolrDocumentSource.java | 22 +-
.../infra/job/archive/SolrQueryBuilder.java | 28 ++-
.../infra/job/archive/SolrQueryProperties.java | 40 +++-
.../infra/job/archive/TarGzCompressor.java | 2 +-
.../src/main/resources/infra-manager.properties | 48 +++-
.../archive/DocumentExportPropertiesTest.java | 54 +++++
.../job/archive/DocumentItemReaderTest.java | 8 +-
.../archive/LocalDocumentItemWriterTest.java | 8 +-
.../infra/job/archive/SolrQueryBuilderTest.java | 18 +-
.../job/archive/SolrQueryPropertiesTest.java | 54 +++++
ambari-infra/pom.xml | 5 +-
39 files changed, 1532 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/pom.xml b/ambari-infra/ambari-infra-manager-it/pom.xml
new file mode 100644
index 0000000..97e8ea0
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/pom.xml
@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <artifactId>ambari-infra</artifactId>
+ <groupId>org.apache.ambari</groupId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ </parent>
+
+ <name>Ambari Infra Manager Integration Tests</name>
+ <url>http://maven.apache.org</url>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>ambari-infra-manager-it</artifactId>
+
+ <properties>
+ <jbehave.version>4.0.5</jbehave.version>
+ <failsafe-plugin.version>2.20</failsafe-plugin.version>
+ <docker.host>localhost</docker.host>
+ <stories.location>NONE</stories.location>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>1.11.5</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.20</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.7.20</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jbehave</groupId>
+ <artifactId>jbehave-core</artifactId>
+ <version>${jbehave.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <testOutputDirectory>target/classes</testOutputDirectory>
+ <testResources>
+ <testResource>
+ <directory>src/test/java/</directory>
+ <includes>
+ <include>**/*.story</include>
+ </includes>
+ </testResource>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>it</id>
+ <activation>
+ <property>
+ <name>it</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${failsafe-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>run-integration-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>**/*Stories.java</include>
+ </includes>
+ <systemPropertyVariables>
+ <log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration>
+ <docker.host>${docker.host}</docker.host>
+ <backend.stories.location>${stories.location}</backend.stories.location>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ <execution>
+ <id>verify-integration-tests</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java
new file mode 100644
index 0000000..0e391a3
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+
+// TODO: use swagger
+public class InfraClient implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(InfraClient.class);
+
+ private final CloseableHttpClient httpClient;
+ private final URI baseUrl;
+
+ public InfraClient(String baseUrl) {
+ try {
+ this.baseUrl = new URI(baseUrl);
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ httpClient = HttpClientBuilder.create().setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)).build();
+ }
+
+ @Override
+ public void close() throws Exception {
+ httpClient.close();
+ }
+
+ // TODO: return job data
+ public void getJobs() {
+ execute(new HttpGet(baseUrl));
+ }
+
+ private String execute(HttpRequestBase post) {
+ try (CloseableHttpResponse response = httpClient.execute(post)) {
+ String responseBodyText = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset());
+ LOG.info("Response code {} body {} ", response.getStatusLine().getStatusCode(), responseBodyText);
+ return responseBodyText;
+ } catch (ClientProtocolException e) {
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ // TODO: return job data
+ public void startJob(String jobName, String parameters) {
+ URIBuilder uriBuilder = new URIBuilder(baseUrl);
+ uriBuilder.setScheme("http");
+ uriBuilder.setPath(uriBuilder.getPath() + "/" + jobName);
+ if (!isBlank(parameters))
+ uriBuilder.addParameter("params", parameters);
+ try {
+ execute(new HttpPost(uriBuilder.build()));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraManagerStories.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraManagerStories.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraManagerStories.java
new file mode 100644
index 0000000..cf720ef
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraManagerStories.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+import com.google.common.collect.Lists;
+import org.apache.ambari.infra.steps.ExportJobsSteps;
+import org.apache.commons.lang.StringUtils;
+import org.jbehave.core.configuration.Configuration;
+import org.jbehave.core.configuration.MostUsefulConfiguration;
+import org.jbehave.core.io.LoadFromClasspath;
+import org.jbehave.core.io.LoadFromRelativeFile;
+import org.jbehave.core.io.StoryFinder;
+import org.jbehave.core.io.StoryLoader;
+import org.jbehave.core.junit.JUnitStories;
+import org.jbehave.core.reporters.Format;
+import org.jbehave.core.reporters.StoryReporterBuilder;
+import org.jbehave.core.steps.InjectableStepsFactory;
+import org.jbehave.core.steps.InstanceStepsFactory;
+import org.jbehave.core.steps.ParameterConverters;
+
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.jbehave.core.io.CodeLocations.codeLocationFromClass;
+
+public class InfraManagerStories extends JUnitStories {
+ private static final String BACKEND_STORIES_LOCATION_PROPERTY = "backend.stories.location";
+ private static final String STORY_SUFFIX = ".story";
+
+ @Override
+ public Configuration configuration() {
+ return new MostUsefulConfiguration()
+ .useStoryLoader(getStoryLoader(BACKEND_STORIES_LOCATION_PROPERTY, this.getClass()))
+ .useParameterConverters(new ParameterConverters().addConverters(new OffsetDateTimeConverter()))
+ .useStoryReporterBuilder(
+ new StoryReporterBuilder().withFailureTrace(true).withDefaultFormats().withFormats(Format.CONSOLE, Format.TXT));
+ }
+
+ private static StoryLoader getStoryLoader(String property, Class clazz) {
+ boolean useExternalStoryLocation = useExternalStoryLocation(property);
+ if (useExternalStoryLocation) {
+ try {
+ return new LoadFromRelativeFile(new URL("file://" + System.getProperty(property)));
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot load story files from url: file://" + System.getProperty(property));
+ }
+ } else {
+ return new LoadFromClasspath(clazz);
+ }
+ }
+
+ @Override
+ public InjectableStepsFactory stepsFactory() {
+ return new InstanceStepsFactory(configuration(), new ExportJobsSteps());
+ }
+
+ @Override
+ protected List<String> storyPaths() {
+ return findStories(BACKEND_STORIES_LOCATION_PROPERTY, STORY_SUFFIX, this.getClass());
+ }
+
+ private static List<String> findStories(String property, String suffix, Class clazz) {
+ if (useExternalStoryLocation(property)) {
+ return findStoriesInFolder(System.getProperty(property), suffix);
+ } else {
+ return new StoryFinder()
+ .findPaths(codeLocationFromClass(clazz).getFile(), singletonList(String.format("**/*%s", suffix)), null);
+ }
+ }
+
+ private static List<String> findStoriesInFolder(String folderAbsolutePath, String suffix) {
+ List<String> results = Lists.newArrayList();
+ File folder = new File(folderAbsolutePath);
+ File[] listOfFiles = folder.listFiles();
+ if (listOfFiles != null) {
+ for (File file : listOfFiles) {
+ if (file.getName().endsWith(suffix)) {
+ results.add(file.getName());
+ }
+ }
+ }
+ return results;
+ }
+
+ private static boolean useExternalStoryLocation(String property) {
+ String storyLocationProp = System.getProperty(property);
+ return StringUtils.isNotEmpty(storyLocationProp) && !"NONE".equals(storyLocationProp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/OffsetDateTimeConverter.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/OffsetDateTimeConverter.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/OffsetDateTimeConverter.java
new file mode 100644
index 0000000..9db562c
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/OffsetDateTimeConverter.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+import org.jbehave.core.steps.ParameterConverters;
+
+import java.lang.reflect.Type;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+
+public class OffsetDateTimeConverter implements ParameterConverters.ParameterConverter {
+ private static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");
+
+ @Override
+ public boolean accept(Type type) {
+ return type instanceof Class<?> && OffsetDateTime.class.isAssignableFrom((Class<?>) type);
+ }
+
+ @Override
+ public Object convertValue(String value, Type type) {
+ return OffsetDateTime.parse(value, SOLR_DATETIME_FORMATTER);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
new file mode 100644
index 0000000..703e1cf
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.steps;
+
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import org.apache.ambari.infra.InfraClient;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.common.SolrInputDocument;
+import org.jbehave.core.annotations.AfterStories;
+import org.jbehave.core.annotations.BeforeStories;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.UUID;
+import java.util.function.BooleanSupplier;
+
+import static java.lang.System.currentTimeMillis;
+
+public abstract class AbstractInfraSteps {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractInfraSteps.class);
+
+ private static final int SOLR_PORT = 8983;
+ private static final int INFRA_MANAGER_PORT = 61890;
+ private static final int FAKE_S3_PORT = 4569;
+ private static final String AUDIT_LOGS_COLLECTION = "audit_logs";
+ protected static final String S3_BUCKET_NAME = "testbucket";
+ private String ambariFolder;
+ private String shellScriptLocation;
+ private String dockerHost;
+ private SolrClient solrClient;
+ private AmazonS3Client s3client;
+
+ public InfraClient getInfraClient() {
+ return new InfraClient(String.format("http://%s:%d/api/v1/jobs", dockerHost, INFRA_MANAGER_PORT));
+ }
+
+ public SolrClient getSolrClient() {
+ return solrClient;
+ }
+
+ public AmazonS3Client getS3client() {
+ return s3client;
+ }
+
+ @BeforeStories
+ public void initDockerContainer() throws Exception {
+ LOG.info("Create new docker container for testing Ambari Infra Manager ...");
+ URL location = AbstractInfraSteps.class.getProtectionDomain().getCodeSource().getLocation();
+ ambariFolder = new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent();
+ shellScriptLocation = ambariFolder + "/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh";
+
+ runCommand(new String[]{shellScriptLocation, "start"});
+
+ dockerHost = System.getProperty("docker.host") != null ? System.getProperty("docker.host") : "localhost";
+
+ waitUntilSolrIsUp();
+
+ solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(String.format("http://%s:%d/solr/%s_shard1_replica1",
+ dockerHost,
+ SOLR_PORT,
+ AUDIT_LOGS_COLLECTION)).build();
+
+ LOG.info("Creating collection");
+ runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-c", AUDIT_LOGS_COLLECTION, "-d", "configsets/"+ AUDIT_LOGS_COLLECTION +"/conf", "-n", AUDIT_LOGS_COLLECTION + "_conf"});
+
+ LOG.info("Initializing s3 client");
+ s3client = new AmazonS3Client(new BasicAWSCredentials("remote-identity", "remote-credential"));
+ s3client.setEndpoint(String.format("http://%s:%d", dockerHost, FAKE_S3_PORT));
+ s3client.createBucket(S3_BUCKET_NAME);
+
+ checkInfraManagerReachable();
+ }
+
+ protected void runCommand(String[] command) {
+ try {
+ LOG.info("Exec command: {}", StringUtils.join(command, " "));
+ Process process = Runtime.getRuntime().exec(command);
+ String stdout = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8);
+ LOG.info("Exec command result {}", stdout);
+ } catch (Exception e) {
+ throw new RuntimeException("Error during execute shell command: ", e);
+ }
+ }
+
+ private void waitUntilSolrIsUp() throws Exception {
+ try(CloseableHttpClient httpClient = HttpClientBuilder.create().setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)).build()) {
+ doWithin(60, "Start Solr", () -> pingSolr(httpClient));
+ }
+ }
+
+ protected void doWithin(int sec, String actionName, BooleanSupplier predicate) {
+ doWithin(sec, actionName, () -> {
+ if (!predicate.getAsBoolean())
+ throw new RuntimeException("Predicate was false!");
+ });
+ }
+
+ protected void doWithin(int sec, String actionName, Runnable runnable) {
+ long start = currentTimeMillis();
+ Exception exception;
+ while (true) {
+ try {
+ runnable.run();
+ return;
+ }
+ catch (Exception e) {
+ exception = e;
+ }
+
+ if (currentTimeMillis() - start > sec * 1000) {
+ throw new AssertionError(String.format("Unable to perform action '%s' within %d seconds", actionName, sec), exception);
+ }
+ else {
+ LOG.info("Performing action '{}' failed. retrying...", actionName);
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private boolean pingSolr(CloseableHttpClient httpClient) {
+ try (CloseableHttpResponse response = httpClient.execute(new HttpGet(String.format("http://%s:%d/solr/admin/collections?action=LIST", dockerHost, SOLR_PORT)))) {
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private void checkInfraManagerReachable() throws Exception {
+ try (InfraClient httpClient = getInfraClient()) {
+ doWithin(30, "Start Ambari Infra Manager", httpClient::getJobs);
+ LOG.info("Ambari Infra Manager is up and running");
+ }
+ }
+
+ protected void addDocument(OffsetDateTime logtime) throws SolrServerException, IOException {
+ SolrInputDocument solrInputDocument = new SolrInputDocument();
+ solrInputDocument.addField("logType", "HDFSAudit");
+ solrInputDocument.addField("cluster", "cl1");
+ solrInputDocument.addField("event_count", 1);
+ solrInputDocument.addField("repo", "hdfs");
+ solrInputDocument.addField("reqUser", "ambari-qa");
+ solrInputDocument.addField("type", "hdfs_audit");
+ solrInputDocument.addField("seq_num", 9);
+ solrInputDocument.addField("result", 1);
+ solrInputDocument.addField("path", "/root/test-logs/hdfs-audit/hdfs-audit.log");
+ solrInputDocument.addField("ugi", "ambari-qa (auth:SIMPLE)");
+ solrInputDocument.addField("host", "logfeeder.apache.org");
+ solrInputDocument.addField("action", "getfileinfo");
+ solrInputDocument.addField("log_message", "allowed=true\tugi=ambari-qa (auth:SIMPLE)\tip=/192.168.64.102\tcmd=getfileinfo\tsrc=/ats/active\tdst=null\tperm=null\tproto=rpc\tcallerContext=HIVE_QUERY_ID:ambari-qa_20160317200111_223b3079-4a2d-431c-920f-6ba37ed63e9f");
+ solrInputDocument.addField("logger_name", "FSNamesystem.audit");
+ solrInputDocument.addField("id", UUID.randomUUID().toString());
+ solrInputDocument.addField("authType", "SIMPLE");
+ solrInputDocument.addField("logfile_line_number", 1);
+ solrInputDocument.addField("cliIP", "/192.168.64.102");
+ solrInputDocument.addField("level", "INFO");
+ solrInputDocument.addField("resource", "/ats/active");
+ solrInputDocument.addField("ip", "172.18.0.2");
+ solrInputDocument.addField("evtTime", "2017-12-08T10:23:16.452Z");
+ solrInputDocument.addField("req_caller_id", "HIVE_QUERY_ID:ambari-qa_20160317200111_223b3079-4a2d-431c-920f-6ba37ed63e9f");
+ solrInputDocument.addField("repoType", 1);
+ solrInputDocument.addField("enforcer", "hadoop-acl");
+ solrInputDocument.addField("cliType", "rpc");
+ solrInputDocument.addField("message_md5", "-6778765776916226588");
+ solrInputDocument.addField("event_md5", "5627261521757462732");
+ solrInputDocument.addField("logtime", new Date(logtime.toInstant().toEpochMilli()));
+ solrInputDocument.addField("_ttl_", "+7DAYS");
+ solrInputDocument.addField("_expire_at_", "2017-12-15T10:23:19.106Z");
+ solrClient.add(solrInputDocument);
+ }
+
+ @AfterStories
+ public void shutdownContainers() throws Exception {
+ Thread.sleep(2000); // sync with s3 server
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+ ObjectListing objectListing = getS3client().listObjects(listObjectsRequest);
+ LOG.info("Found {} files on s3.", objectListing.getObjectSummaries().size());
+ objectListing.getObjectSummaries().forEach(s3ObjectSummary -> LOG.info("Found file in s3 with key {}", s3ObjectSummary.getKey()));
+
+ LOG.info("shutdown containers");
+ runCommand(new String[]{shellScriptLocation, "stop"});
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
new file mode 100644
index 0000000..4a09d7d
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.steps;
+
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.apache.ambari.infra.InfraClient;
+import org.jbehave.core.annotations.Alias;
+import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.Then;
+import org.jbehave.core.annotations.When;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.time.Duration;
+import java.time.OffsetDateTime;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasProperty;
+import static org.hamcrest.core.IsCollectionContaining.hasItem;
+import static org.junit.Assert.assertThat;
+
+public class ExportJobsSteps extends AbstractInfraSteps {
+ private static final Logger LOG = LoggerFactory.getLogger(ExportJobsSteps.class);
+
+ @Given("$count documents in solr")
+ public void addDocuments(int count) throws Exception {
+ for (int i = 0; i < count; ++i)
+ addDocument(OffsetDateTime.now().minusMinutes(i));
+ getSolrClient().commit();
+ }
+
+ @Given("$count documents in solr with logtime from $startLogtime to $endLogtime")
+ public void addDocuments(long count, OffsetDateTime startLogtime, OffsetDateTime endLogtime) throws Exception {
+ Duration duration = Duration.between(startLogtime, endLogtime);
+ long increment = duration.toNanos() / count;
+ for (int i = 0; i < count; ++i)
+ addDocument(startLogtime.plusNanos(increment * i));
+ getSolrClient().commit();
+ }
+
+ @Given("a file on s3 with key $key")
+ public void addFileToS3(String key) throws Exception {
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream("anything".getBytes())) {
+ getS3client().putObject(S3_BUCKET_NAME, key, inputStream, new ObjectMetadata());
+ }
+ }
+
+ @When("start $jobName job")
+ public void startJob(String jobName) throws Exception {
+ startJob(jobName, null);
+ }
+
+ @When("start $jobName job with parameters $parameters")
+ @Alias("restart $jobName job with parameters $parameters")
+ public void startJob(String jobName, String parameters) throws Exception {
+ try (InfraClient httpClient = getInfraClient()) {
+ httpClient.startJob(jobName, parameters);
+ }
+ }
+
+ @When("delete file with key $key from s3")
+ public void deleteFileFromS3(String key) {
+ getS3client().deleteObject(S3_BUCKET_NAME, key);
+ }
+
+ @Then("Check filenames contains the text $text on s3 server after $waitSec seconds")
+ public void checkS3After(String text, int waitSec) throws Exception {
+ AmazonS3Client s3Client = getS3client();
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+ doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
+ && !s3Client.listObjects(listObjectsRequest).getObjectSummaries().isEmpty());
+
+ ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
+ assertThat(objectListing.getObjectSummaries(), hasItem(hasProperty("key", containsString(text))));
+ }
+
+ @Then("Check $count files exists on s3 server with filenames containing the text $text after $waitSec seconds")
+ public void checkNumberOfFilesOnS3(int count, String text, int waitSec) {
+ AmazonS3Client s3Client = getS3client();
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+ doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
+ && s3Client.listObjects(listObjectsRequest).getObjectSummaries().stream()
+ .filter(s3ObjectSummary -> s3ObjectSummary.getKey().contains(text))
+ .count() == count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/resources/log4j.properties b/ambari-infra/ambari-infra-manager-it/src/test/resources/log4j.properties
new file mode 100644
index 0000000..956bc63
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/resources/log4j.properties
@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story b/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
new file mode 100644
index 0000000..cd1f49d
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
@@ -0,0 +1,23 @@
+Scenario: Export documents form solr and upload them to s3 using defult configuration
+
+Given 1000 documents in solr
+When start export_audit_logs job
+Then Check filenames contains the text audit_logs on s3 server after 20 seconds
+
+
+Scenario: Exporting 10 documents using writeBlockSize=3 produces 4 files
+
+Given 10 documents in solr with logtime from 2010-10-09T05:00:00.000Z to 2010-10-09T20:00:00.000Z
+When start export_audit_logs job with parameters writeBlockSize=3,start=2010-10-09T00:00:00.000Z,end=2010-10-11T00:00:00.000Z
+Then Check 4 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2010-10-09 after 20 seconds
+
+
+Scenario: Export job fails when part of the data is exported. After resolving the issue and restarting the job exports the rest of the data.
+
+Given 200 documents in solr with logtime from 2011-10-09T05:00:00.000Z to 2011-10-09T20:00:00.000Z
+And a file on s3 with key solr_archive_audit_logs_-_2011-10-09T08:00:00.000Z.json.tar.gz
+When start export_audit_logs job with parameters writeBlockSize=20,start=2010-11-09T00:00:00.000Z,end=2011-10-11T00:00:00.000Z
+Then Check 3 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
+When delete file with key solr_archive_audit_logs_-_2011-10-09T08:00:00.000Z.json.tar.gz from s3
+And restart export_audit_logs job with parameters writeBlockSize=20,start=2010-11-09T00:00:00.000Z,end=2011-10-11T00:00:00.000Z
+Then Check 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/docker/Dockerfile b/ambari-infra/ambari-infra-manager/docker/Dockerfile
index adb584a..eaefe95 100644
--- a/ambari-infra/ambari-infra-manager/docker/Dockerfile
+++ b/ambari-infra/ambari-infra-manager/docker/Dockerfile
@@ -22,9 +22,9 @@ RUN yum -y install glibc-common
ENV HOME /root
#Install JAVA
-ENV JAVA_VERSION 8u31
-ENV BUILD_VERSION b13
-RUN wget --no-cookies --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/$JAVA_VERSION-$BUILD_VERSION/jdk-$JAVA_VERSION-linux-x64.rpm" -O jdk-8-linux-x64.rpm
+ENV JAVA_VERSION 8u131
+ENV BUILD_VERSION b11
+RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/$JAVA_VERSION-$BUILD_VERSION/d54c1d3a095b4ff2b6607d096fa80163/jdk-$JAVA_VERSION-linux-x64.rpm -O jdk-8-linux-x64.rpm
RUN rpm -ivh jdk-8-linux-x64.rpm
ENV JAVA_HOME /usr/java/default/
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/docker/docker-compose.yml b/ambari-infra/ambari-infra-manager/docker/docker-compose.yml
new file mode 100644
index 0000000..1172631
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/docker/docker-compose.yml
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+version: '3.3'
+services:
+ zookeeper:
+ image: zookeeper:${ZOOKEEPER_VERSION:-3.4.10}
+ restart: always
+ hostname: zookeeper
+ networks:
+ - infra-network
+ ports:
+ - 2181:2181
+ environment:
+ ZOO_MY_ID: 1
+ ZOO_SERVERS: server.1=zookeeper:2888:3888
+ solr:
+ image: solr:${SOLR_VERSION:-6.6.2}
+ restart: always
+ hostname: solr
+ ports:
+ - "8983:8983"
+ networks:
+ - infra-network
+ env_file:
+ - Profile
+ entrypoint:
+ - docker-entrypoint.sh
+ - solr
+ - start
+ - "-f"
+ - "-c"
+ - "-z"
+ - ${ZOOKEEPER_CONNECTION_STRING}
+ volumes:
+ - $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-server/src/main/configsets:/opt/solr/configsets
+ localstack-s3:
+ image: localstack/localstack
+ ports:
+ - "4569:4569"
+ environment:
+ - SERVICES=s3:4569
+ hostname: fakes3
+ networks:
+ infra-network:
+ aliases:
+ - testbucket.fakes3
+ env_file:
+ - Profile
+ inframanager:
+ image: ambari-infra-manager:v1.0
+ restart: always
+ hostname: infra-manager.apache.org
+ networks:
+ - infra-network
+ env_file:
+ - Profile
+ ports:
+ - 61890:61890
+ - 5007:5007
+ environment:
+ COMPONENT: infra-manager
+ COMPONENT_LOG: infra-manager
+ ZK_CONNECT_STRING: ${ZOOKEEPER_CONNECTION_STRING}
+ DISPLAY: $DOCKERIP:0
+ volumes:
+ - $AMBARI_LOCATION/ambari-infra/ambari-infra-manager/target/package:/root/ambari-infra-manager
+networks:
+ infra-network:
+ driver: bridge
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh b/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh
new file mode 100644
index 0000000..ab02659
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+sdir="`dirname \"$0\"`"
+: ${1:?"argument is missing: (start|stop)"}
+command="$1"
+
+function start_containers() {
+ check_env_files
+ echo "Start containers ..."
+ pushd $sdir/../
+ local AMBARI_INFRA_MANAGER_LOCATION=$(pwd)
+ echo $AMBARI_INFRA_MANAGER_LOCATION
+ kill_containers
+ cd $AMBARI_INFRA_MANAGER_LOCATION/docker
+ docker-compose up -d
+ popd
+ echo "Containers started"
+}
+
+function check_env_files() {
+ local count=0;
+
+ check_env_file .env setup_env
+ count=$((count + $?));
+ check_env_file Profile setup_profile
+ count=$((count + $?));
+
+ if [[ "$count" -gt 0 ]]
+ then
+ echo "Exit"
+ exit;
+ fi
+}
+
+function check_env_file() {
+ if [ -f "$sdir/$1" ];
+ then
+ echo "$1 file exists"
+ return 0;
+ else
+ echo "$1 file does not exist, Creating a new one..."
+ $2
+ echo "$1 file has been created. Check it out before starting Ambari Infra Manager. ($sdir/$1)"
+ return 1;
+ fi
+}
+
+function setup_env() {
+ pushd $sdir/../../
+ local AMBARI_LOCATION=$(pwd)
+ popd
+ local docker_ip=$(get_docker_ip)
+ cat << EOF > $sdir/.env
+DOCKERIP=$docker_ip
+MAVEN_REPOSITORY_LOCATION=$HOME/.m2
+AMBARI_LOCATION=$AMBARI_LOCATION
+
+ZOOKEEPER_VERSION=3.4.10
+ZOOKEEPER_CONNECTION_STRING=zookeeper:2181
+
+SOLR_VERSION=6.6.2
+EOF
+}
+
+function setup_profile() {
+ pushd $sdir/../../
+ local AMBARI_LOCATION=$(pwd)
+ popd
+ cat << EOF > $sdir/Profile
+EOF
+}
+
+function kill_containers() {
+ echo "Try to remove containers if exists ..."
+ docker rm -f docker_inframanager_1
+ docker rm -f docker_solr_1
+ docker rm -f docker_zookeeper_1
+ docker rm -f docker_localstack-s3_1
+}
+
+case $command in
+ "start")
+ start_containers
+ ;;
+ "stop")
+ kill_containers
+ ;;
+ *)
+ echo "Available commands: (start|stop|build-and-run|build|build-docker-and-run|build-mvn-and-run|build-docker-only|build-mvn-only)"
+ ;;
+esac
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/ObjectSource.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/ObjectSource.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/ObjectSource.java
new file mode 100644
index 0000000..98a1e0d
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/ObjectSource.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job;
+
+public interface ObjectSource<T> {
+ CloseableIterator<T> open(T current, int rows);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/AbstractFileAction.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/AbstractFileAction.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/AbstractFileAction.java
new file mode 100644
index 0000000..7a30393
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/AbstractFileAction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+import java.io.File;
+
+public abstract class AbstractFileAction implements FileAction {
+ @Override
+ public File perform(File inputFile, boolean deleteInput) {
+ File outputFile = perform(inputFile);
+ if (deleteInput)
+ inputFile.delete();
+ return outputFile;
+ }
+
+ protected abstract File perform(File inputFile);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/CompositeFileAction.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/CompositeFileAction.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/CompositeFileAction.java
index 84ce160..8421802 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/CompositeFileAction.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/CompositeFileAction.java
@@ -19,6 +19,7 @@
package org.apache.ambari.infra.job.archive;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import static java.util.Arrays.asList;
@@ -28,7 +29,7 @@ public class CompositeFileAction implements FileAction {
private final List<FileAction> actions;
public CompositeFileAction(FileAction... actions) {
- this.actions = asList(actions);
+ this.actions = new ArrayList<>(asList(actions));
}
public void add(FileAction action) {
@@ -36,10 +37,10 @@ public class CompositeFileAction implements FileAction {
}
@Override
- public File perform(File inputFile) {
+ public File perform(File inputFile, boolean deleteInput) {
File file = inputFile;
for (FileAction action : actions) {
- file = action.perform(file);
+ file = action.perform(file, deleteInput);
}
return file;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/Document.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/Document.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/Document.java
index 84f5ece..1f3957a 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/Document.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/Document.java
@@ -26,7 +26,6 @@ import java.util.Map;
import static java.util.Collections.unmodifiableMap;
-// TODO: create entities for each solr collections
public class Document {
private final Map<String, String> fieldMap;
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportConfiguration.java
index 69f41d3..1895911 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportConfiguration.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportConfiguration.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.ObjectSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
@@ -26,28 +27,23 @@ import org.springframework.batch.core.configuration.annotation.JobBuilderFactory
import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.io.File;
import java.nio.file.Paths;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-
-import static org.apache.ambari.infra.job.archive.SolrDocumentSource.SOLR_DATETIME_FORMATTER;
-import static org.apache.commons.lang.StringUtils.isBlank;
@Configuration
public class DocumentExportConfiguration {
private static final Logger LOG = LoggerFactory.getLogger(DocumentExportConfiguration.class);
- private static final DateTimeFormatter FILENAME_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH_mm_ss.SSSX");
@Inject
- private DocumentExportProperties properties;
+ private DocumentExportPropertyMap propertyMap;
@Inject
private StepBuilderFactory steps;
@@ -55,11 +51,26 @@ public class DocumentExportConfiguration {
@Inject
private JobBuilderFactory jobs;
+ @Inject
+ @Qualifier("exportStep")
+ private Step exportStep;
+
+ @Inject
+ private JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor;
- @Bean
- public Job logExportJob(@Qualifier("exportStep") Step logExportStep) {
- return jobs.get("solr_data_export").listener(new DocumentExportJobListener()).start(logExportStep).build();
+ @PostConstruct
+ public void createJobs() {
+ propertyMap.getSolrDataExport().values().forEach(DocumentExportProperties::validate);
+
+ propertyMap.getSolrDataExport().keySet().forEach(jobName -> {
+ Job job = logExportJob(jobName, exportStep);
+ jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
+ });
+ }
+
+ private Job logExportJob(String jobName, Step logExportStep) {
+ return jobs.get(jobName).listener(new DocumentExportJobListener(propertyMap)).start(logExportStep).build();
}
@Bean
@@ -67,16 +78,17 @@ public class DocumentExportConfiguration {
public Step exportStep(DocumentExporter documentExporter) {
return steps.get("export")
.tasklet(documentExporter)
- .listener(new DocumentExportStepListener(properties))
.build();
}
@Bean
@StepScope
- public DocumentExporter getDocumentExporter(DocumentItemReader documentItemReader,
- @Value("#{stepExecution.jobExecution.id}") String jobId) {
+ public DocumentExporter documentExporter(DocumentItemReader documentItemReader,
+ @Value("#{stepExecution.jobExecution.id}") String jobId,
+ @Value("#{stepExecution.jobExecution.executionContext.get('exportProperties')}") DocumentExportProperties properties) {
File path = Paths.get(
properties.getDestinationDirectoryPath(),
+ // TODO: jobId should remain the same after continuing job
String.format("%s_%s", properties.getQuery().getCollection(), jobId)).toFile(); // TODO: add end date
LOG.info("Destination directory path={}", path);
if (!path.exists()) {
@@ -86,33 +98,43 @@ public class DocumentExportConfiguration {
}
CompositeFileAction fileAction = new CompositeFileAction(new TarGzCompressor());
+ properties.s3Properties().ifPresent(s3Properties -> fileAction.add(new S3Uploader(s3Properties)));
return new DocumentExporter(
documentItemReader,
- firstDocument -> new LocalDocumentItemWriter(
- new File(path, String.format("%s_-_%s.json",
- properties.getQuery().getCollection(),
- firstDocument.get(properties.getFileNameSuffixColumn()))),
- fileAction),
+ firstDocument -> localDocumentItemWriter(properties, path, fileAction, firstDocument),
properties.getWriteBlockSize());
}
+ private LocalDocumentItemWriter localDocumentItemWriter(DocumentExportProperties properties, File path, FileAction fileAction, Document firstDocument) {
+ return new LocalDocumentItemWriter(outFile(properties.getQuery().getCollection(), path, firstDocument.get(properties.getFileNameSuffixColumn())),
+ file -> fileAction.perform(file, true));
+ }
+
+ private File outFile(String collection, File directoryPath, String suffix) {
+ // TODO: format date (suffix)
+ File file = new File(directoryPath, String.format("%s_-_%s.json", collection, suffix));
+ LOG.info("Exporting to temp file {}", file.getAbsolutePath());
+ return file;
+ }
+
@Bean
@StepScope
- public DocumentItemReader reader(DocumentSource documentSource) {
+ public DocumentItemReader reader(ObjectSource<Document> documentSource,
+ @Value("#{stepExecution.jobExecution.executionContext.get('exportProperties')}") DocumentExportProperties properties) {
return new DocumentItemReader(documentSource, properties.getReadBlockSize());
}
@Bean
@StepScope
- public DocumentSource logSource(@Value("#{jobParameters[endDate]}") String endDateText) {
- OffsetDateTime endDate = OffsetDateTime.now(ZoneOffset.UTC);
- if (!isBlank(endDateText))
- endDate = OffsetDateTime.parse(endDateText);
+ public ObjectSource logSource(@Value("#{jobParameters[start]}") String start,
+ @Value("#{jobParameters[end]}") String end,
+ @Value("#{stepExecution.jobExecution.executionContext.get('exportProperties')}") DocumentExportProperties properties) {
return new SolrDocumentSource(
- properties.getZooKeeperSocket(),
+ properties.getZooKeeperConnectionString(),
properties.getQuery(),
- SOLR_DATETIME_FORMATTER.format(endDate));
+ start,
+ end);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportJobListener.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportJobListener.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportJobListener.java
index f1df46c..3b6c402 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportJobListener.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportJobListener.java
@@ -23,9 +23,32 @@ import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionListener;
public class DocumentExportJobListener implements JobExecutionListener {
+
+ private final DocumentExportPropertyMap propertyMap;
+
+ public DocumentExportJobListener(DocumentExportPropertyMap propertyMap) {
+ this.propertyMap = propertyMap;
+ }
+
+
@Override
public void beforeJob(JobExecution jobExecution) {
+ try {
+ String jobName = jobExecution.getJobInstance().getJobName();
+ DocumentExportProperties defaultProperties = propertyMap.getSolrDataExport().get(jobName);
+ if (defaultProperties == null)
+ throw new UnsupportedOperationException("Properties not found for job " + jobName);
+ DocumentExportProperties properties = defaultProperties.deepCopy();
+ properties.apply(jobExecution.getJobParameters());
+ properties.validate();
+ jobExecution.getExecutionContext().put("exportProperties", properties);
+ }
+ catch (UnsupportedOperationException | IllegalArgumentException ex) {
+ jobExecution.stop();
+ jobExecution.setExitStatus(new ExitStatus(ExitStatus.FAILED.getExitCode(), ex.getMessage()));
+ throw ex;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java
index d6301c0..37f6d1b 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java
@@ -18,38 +18,34 @@
*/
package org.apache.ambari.infra.job.archive;
-import org.hibernate.validator.constraints.NotBlank;
+import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.batch.core.JobParameters;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.PropertySource;
-import javax.validation.constraints.Min;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Optional;
import static org.apache.commons.lang.StringUtils.isBlank;
-@Configuration
-@PropertySource(value = {"classpath:infra-manager.properties"})
-@ConfigurationProperties(prefix = "infra-manager.jobs.solr_data_export")
public class DocumentExportProperties {
- @NotBlank
- private String zooKeeperSocket;
- @Min(1)
+ private String zooKeeperConnectionString;
private int readBlockSize;
- @Min(1)
private int writeBlockSize;
- @NotBlank
private String destinationDirectoryPath;
- @NotBlank
private String fileNameSuffixColumn;
private SolrQueryProperties query;
-
- public String getZooKeeperSocket() {
- return zooKeeperSocket;
+ private String s3AccessKey;
+ private String s3SecretKey;
+ private String s3KeyPrefix;
+ private String s3BucketName;
+ private String s3Endpoint;
+
+ public String getZooKeeperConnectionString() {
+ return zooKeeperConnectionString;
}
- public void setZooKeeperSocket(String zooKeeperSocket) {
- this.zooKeeperSocket = zooKeeperSocket;
+ public void setZooKeeperConnectionString(String zooKeeperConnectionString) {
+ this.zooKeeperConnectionString = zooKeeperConnectionString;
}
public int getReadBlockSize() {
@@ -76,37 +72,109 @@ public class DocumentExportProperties {
this.destinationDirectoryPath = destinationDirectoryPath;
}
+ public String getFileNameSuffixColumn() {
+ return fileNameSuffixColumn;
+ }
+
+ public void setFileNameSuffixColumn(String fileNameSuffixColumn) {
+ this.fileNameSuffixColumn = fileNameSuffixColumn;
+ }
+
+ public SolrQueryProperties getQuery() {
+ return query;
+ }
+
+ public void setQuery(SolrQueryProperties query) {
+ this.query = query;
+ }
+
+ public String getS3AccessKey() {
+ return s3AccessKey;
+ }
+
+ public void setS3AccessKey(String s3AccessKey) {
+ this.s3AccessKey = s3AccessKey;
+ }
+
+ public String getS3SecretKey() {
+ return s3SecretKey;
+ }
+
+ public void setS3SecretKey(String s3SecretKey) {
+ this.s3SecretKey = s3SecretKey;
+ }
+
+ public String getS3KeyPrefix() {
+ return s3KeyPrefix;
+ }
+
+ public void setS3KeyPrefix(String s3KeyPrefix) {
+ this.s3KeyPrefix = s3KeyPrefix;
+ }
+
+ public String getS3BucketName() {
+ return s3BucketName;
+ }
+
+ public void setS3BucketName(String s3BucketName) {
+ this.s3BucketName = s3BucketName;
+ }
+
+ public String getS3Endpoint() {
+ return s3Endpoint;
+ }
+
+ public void setS3Endpoint(String s3Endpoint) {
+ this.s3Endpoint = s3Endpoint;
+ }
+
public void apply(JobParameters jobParameters) {
- // TODO: solr query params
- zooKeeperSocket = jobParameters.getString("zooKeeperSocket", zooKeeperSocket);
+ zooKeeperConnectionString = jobParameters.getString("zooKeeperConnectionString", zooKeeperConnectionString);
readBlockSize = getIntJobParameter(jobParameters, "readBlockSize", readBlockSize);
writeBlockSize = getIntJobParameter(jobParameters, "writeBlockSize", writeBlockSize);
destinationDirectoryPath = jobParameters.getString("destinationDirectoryPath", destinationDirectoryPath);
- query.setCollection(jobParameters.getString("collection", query.getCollection()));
- query.setQueryText(jobParameters.getString("queryText", query.getQueryText()));
- query.setFilterQueryText(jobParameters.getString("filterQueryText", query.getFilterQueryText()));
+ query.apply(jobParameters);
}
private int getIntJobParameter(JobParameters jobParameters, String parameterName, int defaultValue) {
- String writeBlockSizeText = jobParameters.getString(parameterName);
- if (isBlank(writeBlockSizeText))
+ String valueText = jobParameters.getString(parameterName);
+ if (isBlank(valueText))
return defaultValue;
- return this.writeBlockSize = Integer.parseInt(writeBlockSizeText);
+ return Integer.parseInt(valueText);
}
- public String getFileNameSuffixColumn() {
- return fileNameSuffixColumn;
+ public DocumentExportProperties deepCopy() {
+ try {
+ ObjectMapper objectMapper = new ObjectMapper();
+ String json = objectMapper.writeValueAsString(this);
+ return objectMapper.readValue(json, DocumentExportProperties.class);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
}
- public void setFileNameSuffixColumn(String fileNameSuffixColumn) {
- this.fileNameSuffixColumn = fileNameSuffixColumn;
+ public Optional<S3Properties> s3Properties() {
+ if (!isBlank(s3AccessKey) && !isBlank(s3SecretKey) && !isBlank(s3BucketName))
+ return Optional.of(new S3Properties(s3AccessKey, s3SecretKey, s3KeyPrefix, s3BucketName, s3Endpoint));
+ return Optional.empty();
}
- public SolrQueryProperties getQuery() {
- return query;
- }
+ public void validate() {
+ if (isBlank(zooKeeperConnectionString))
+ throw new IllegalArgumentException("The property zooKeeperConnectionString can not be null or empty string!");
- public void setQuery(SolrQueryProperties query) {
- this.query = query;
+ if (readBlockSize == 0)
+ throw new IllegalArgumentException("The property readBlockSize must be greater than 0!");
+
+ if (writeBlockSize == 0)
+ throw new IllegalArgumentException("The property writeBlockSize must be greater than 0!");
+
+ if (isBlank(destinationDirectoryPath))
+ throw new IllegalArgumentException("The property destinationDirectoryPath can not be null or empty string!");
+
+ if (isBlank(fileNameSuffixColumn))
+ throw new IllegalArgumentException("The property fileNameSuffixColumn can not be null or empty string!");
+
+ query.validate();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportPropertyMap.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportPropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportPropertyMap.java
new file mode 100644
index 0000000..9af4afc
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportPropertyMap.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.Map;
+
+@Configuration
+@ConfigurationProperties(prefix = "infra-manager.jobs")
+public class DocumentExportPropertyMap {
+ private Map<String, DocumentExportProperties> solrDataExport;
+
+ public Map<String, DocumentExportProperties> getSolrDataExport() {
+ return solrDataExport;
+ }
+
+ public void setSolrDataExport(Map<String, DocumentExportProperties> solrDataExport) {
+ this.solrDataExport = solrDataExport;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportStepListener.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportStepListener.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportStepListener.java
deleted file mode 100644
index 3bab6d5..0000000
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportStepListener.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.infra.job.archive;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.batch.core.ExitStatus;
-import org.springframework.batch.core.StepExecution;
-import org.springframework.batch.core.StepExecutionListener;
-
-public class DocumentExportStepListener implements StepExecutionListener {
- private static final Logger LOG = LoggerFactory.getLogger(DocumentExportStepListener.class);
-
- private final DocumentExportProperties properties;
-
- public DocumentExportStepListener(DocumentExportProperties properties) {
- this.properties = properties;
- }
-
- @Override
- public void beforeStep(StepExecution stepExecution) {
- properties.apply(stepExecution.getJobParameters());
- LOG.info("LogExport step - before step execution");
- }
-
- @Override
- public ExitStatus afterStep(StepExecution stepExecution) {
- LOG.info("LogExport step - after step execution");
- return stepExecution.getExitStatus();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentItemReader.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentItemReader.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentItemReader.java
index a4378a4..3a6b869 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentItemReader.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentItemReader.java
@@ -18,6 +18,8 @@
*/
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.CloseableIterator;
+import org.apache.ambari.infra.job.ObjectSource;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemStreamException;
import org.springframework.batch.item.support.AbstractItemStreamItemReader;
@@ -31,16 +33,16 @@ public class DocumentItemReader extends AbstractItemStreamItemReader<Document> i
public final static String POSITION = "last-read";
- private final DocumentSource documentSource;
+ private final ObjectSource<Document> documentSource;
private final int readBlockSize;
- private DocumentIterator documentIterator = null;
+ private CloseableIterator<Document> documentIterator = null;
private int count = 0;
private boolean eof = false;
private Document current = null;
private Document previous = null;
- public DocumentItemReader(DocumentSource documentSource, int readBlockSize) {
+ public DocumentItemReader(ObjectSource<Document> documentSource, int readBlockSize) {
this.documentSource = documentSource;
this.readBlockSize = readBlockSize;
setName(ClassUtils.getShortName(DocumentItemReader.class));
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentIterator.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentIterator.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentIterator.java
index 6232cfc..5fa29b0 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentIterator.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentIterator.java
@@ -16,10 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job.archive;
+package org.apache.ambari.infra.job;
import java.util.Iterator;
-// TODO: generic closeable iterator
-public interface DocumentIterator extends Iterator<Document>, AutoCloseable {
+public interface CloseableIterator<T> extends Iterator<T>, AutoCloseable {
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentSource.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentSource.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentSource.java
index c9871a3..7427771 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentSource.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentSource.java
@@ -18,7 +18,8 @@
*/
package org.apache.ambari.infra.job.archive;
-// TODO: generic object source
-public interface DocumentSource {
- DocumentIterator open(Document current, int rows);
+import java.io.File;
+
+public interface ItemWriterListener {
+ void onCompleted(File file);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileAction.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileAction.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileAction.java
index 26a8c63..d3f2a65 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileAction.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileAction.java
@@ -21,5 +21,5 @@ package org.apache.ambari.infra.job.archive;
import java.io.File;
public interface FileAction {
- File perform(File inputFile);
+ File perform(File inputFile, boolean deleteInput);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriter.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriter.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriter.java
index 02d898d..baad61b 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriter.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriter.java
@@ -29,10 +29,10 @@ public class LocalDocumentItemWriter implements DocumentItemWriter {
private final File outFile;
private final BufferedWriter bufferedWriter;
- private final FileAction fileAction;
+ private final ItemWriterListener itemWriterListener;
- public LocalDocumentItemWriter(File outFile, FileAction fileAction) {
- this.fileAction = fileAction;
+ public LocalDocumentItemWriter(File outFile, ItemWriterListener itemWriterListener) {
+ this.itemWriterListener = itemWriterListener;
this.outFile = outFile;
try {
this.bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFile), ENCODING));
@@ -64,7 +64,7 @@ public class LocalDocumentItemWriter implements DocumentItemWriter {
public void close() {
try {
bufferedWriter.close();
- fileAction.perform(outFile);
+ itemWriterListener.onCompleted(outFile);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
index 495401d..0979f10 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
@@ -18,47 +18,48 @@
*/
package org.apache.ambari.infra.job.archive;
-import org.hibernate.validator.constraints.NotBlank;
-
public class S3Properties {
- @NotBlank
- private String accessKey;
- @NotBlank
- private String secretKey;
- @NotBlank
- private String keyPrefix;
- @NotBlank
- private String bucketName;
-
- public String getAccessKey() {
- return accessKey;
- }
+ private String s3AccessKey;
+ private String s3SecretKey;
+ private String s3KeyPrefix;
+ private String s3BucketName;
+ private String s3EndPoint;
- public String getSecretKey() {
- return secretKey;
+ public S3Properties(String s3AccessKey, String s3SecretKey, String s3KeyPrefix, String s3BucketName, String s3EndPoint) {
+ this.s3AccessKey = s3AccessKey;
+ this.s3SecretKey = s3SecretKey;
+ this.s3KeyPrefix = s3KeyPrefix;
+ this.s3BucketName = s3BucketName;
+ this.s3EndPoint = s3EndPoint;
}
- public String getKeyPrefix() {
- return keyPrefix;
+ public String getS3AccessKey() {
+ return s3AccessKey;
}
- public String getBucketName() {
- return bucketName;
+ public String getS3SecretKey() {
+ return s3SecretKey;
}
- public void setAccessKey(String accessKey) {
- this.accessKey = accessKey;
+ public String getS3KeyPrefix() {
+ return s3KeyPrefix;
}
- public void setSecretKey(String secretKey) {
- this.secretKey = secretKey;
+ public String getS3BucketName() {
+ return s3BucketName;
}
- public void setKeyPrefix(String keyPrefix) {
- this.keyPrefix = keyPrefix;
+ public String getS3EndPoint() {
+ return s3EndPoint;
}
- public void setBucketName(String bucketName) {
- this.bucketName = bucketName;
+ @Override
+ public String toString() {
+ return "S3Properties{" +
+ "s3AccessKey='" + s3AccessKey + '\'' +
+ ", s3KeyPrefix='" + s3KeyPrefix + '\'' +
+ ", s3BucketName='" + s3BucketName + '\'' +
+ ", s3EndPoint='" + s3EndPoint + '\'' +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
index 3214e50..deeb9c7 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
@@ -2,9 +2,13 @@ package org.apache.ambari.infra.job.archive;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
+import static org.apache.commons.lang.StringUtils.isBlank;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -23,17 +27,25 @@ import java.io.File;
* specific language governing permissions and limitations
* under the License.
*/
-public class S3Uploader implements FileAction {
+public class S3Uploader extends AbstractFileAction {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DocumentExportConfiguration.class);
private final AmazonS3Client client;
private final String keyPrefix;
private final String bucketName;
public S3Uploader(S3Properties s3Properties) {
- this.keyPrefix = s3Properties.getKeyPrefix();
- this.bucketName = s3Properties.getBucketName();
- BasicAWSCredentials credentials = new BasicAWSCredentials(s3Properties.getAccessKey(), s3Properties.getSecretKey());
+ LOG.info("Initializing S3 client with " + s3Properties);
+
+ this.keyPrefix = s3Properties.getS3KeyPrefix();
+ this.bucketName = s3Properties.getS3BucketName();
+ BasicAWSCredentials credentials = new BasicAWSCredentials(s3Properties.getS3AccessKey(), s3Properties.getS3SecretKey());
client = new AmazonS3Client(credentials);
+ if (!isBlank(s3Properties.getS3EndPoint()))
+ client.setEndpoint(s3Properties.getS3EndPoint());
+// Note: without pathStyleAccess=true endpoint going to be <bucketName>.<host>:<port>
+// client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
}
@Override
@@ -41,8 +53,7 @@ public class S3Uploader implements FileAction {
String key = keyPrefix + inputFile.getName();
if (client.doesObjectExist(bucketName, key)) {
- System.out.println("Object '" + key + "' already exists");
- System.exit(0);
+ throw new UnsupportedOperationException(String.format("Object '%s' already exists in bucket '%s'", key, bucketName));
}
client.putObject(bucketName, key, inputFile);
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentIterator.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentIterator.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentIterator.java
index db4069b..2e7341d 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentIterator.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentIterator.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.CloseableIterator;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
@@ -31,7 +32,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.TimeZone;
-public class SolrDocumentIterator implements DocumentIterator {
+public class SolrDocumentIterator implements CloseableIterator<Document> {
private static final DateFormat SOLR_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
[33/51] [abbrv] ambari git commit: AMBARI-22681 Ambari web UI failed
to update stack repo directly from the Versions tab (dili)
Posted by am...@apache.org.
AMBARI-22681 Ambari web UI failed to update stack repo directly from the Versions tab (dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/26bcc97b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/26bcc97b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/26bcc97b
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 26bcc97bcb4016948fd28d76943eb9ad23c7a4e6
Parents: 67fc4a3
Author: Di Li <di...@apache.org>
Authored: Thu Dec 21 11:45:33 2017 -0500
Committer: Di Li <di...@apache.org>
Committed: Thu Dec 21 11:45:33 2017 -0500
----------------------------------------------------------------------
.../app/controllers/main/admin/stack_and_upgrade_controller.js | 1 +
ambari-web/app/utils/ajax/ajax.js | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/26bcc97b/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js b/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
index 2f0cb68..35041bf 100644
--- a/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
+++ b/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
@@ -1695,6 +1695,7 @@ App.MainAdminStackAndUpgradeController = Em.Controller.extend(App.LocalStorage,
sender: this,
data: {
repo: repo,
+ repoName: repo.get('repoName'),
repoId: repo.get('repoId'),
baseUrl: repo.get('baseUrl'),
osType: os.get('osType'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/26bcc97b/ambari-web/app/utils/ajax/ajax.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/ajax/ajax.js b/ambari-web/app/utils/ajax/ajax.js
index 3f85dfdc..42bb974 100644
--- a/ambari-web/app/utils/ajax/ajax.js
+++ b/ambari-web/app/utils/ajax/ajax.js
@@ -1843,7 +1843,8 @@ var urls = {
return {
data: JSON.stringify({
"Repositories": {
- "base_url": data.baseUrl
+ "base_url": data.baseUrl,
+ "repo_name": data.repoName
}
})
}
[06/51] [abbrv] ambari git commit: AMBARI-22635: Addendum fix Ambari
should create a dummy core-site.xml for Ranger plugins when namenode is not
installed.
Posted by am...@apache.org.
AMBARI-22635: Addendum fix Ambari should create a dummy core-site.xml for Ranger plugins when namenode is not installed.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f1f73022
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f1f73022
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f1f73022
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: f1f730229fe559439189726c0cd40cd58cbf0135
Parents: d95d3eb
Author: Vishal Suvagia <vi...@apache.org>
Authored: Fri Dec 15 11:20:31 2017 +0530
Committer: Vishal Suvagia <vi...@apache.org>
Committed: Fri Dec 15 11:20:31 2017 +0530
----------------------------------------------------------------------
.../KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f73022/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
index e2cfbcf..16eff94 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
@@ -90,7 +90,7 @@ def setup_ranger_kafka():
else:
Logger.info("Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations")
setup_core_site_for_required_plugins(component_user = params.kafka_user, component_group = params.user_group,
- create_core_site_path = params.conf_dir, configurations = { 'hadoop.security.authentication' : 'kerberos' if params.security_enabled else 'simple' },
+ create_core_site_path = params.conf_dir, configurations = { 'hadoop.security.authentication' : 'kerberos' if params.kerberos_security_enabled else 'simple' },
configuration_attributes = {})
else:
[49/51] [abbrv] ambari git commit: AMBARI-22115. Alerts for OneFS
mpack (amagyar)
Posted by am...@apache.org.
AMBARI-22115. Alerts for OneFS mpack (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18785175
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18785175
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18785175
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 187851756ab52fcccb7fe40d05617c64162e372d
Parents: 92f8ee0
Author: Attila Magyar <am...@hortonworks.com>
Authored: Thu Oct 26 11:49:18 2017 +0200
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../server/controller/jmx/JMXMetricHolder.java | 55 +++-
.../ambari/server/state/alert/Reporting.java | 9 +-
.../controller/jmx/JMXMetricHolderTest.java | 14 +
.../addon-services/ONEFS/1.0.0/alerts.json | 112 ++++++--
.../ONEFS/1.0.0/configuration/hadoop-env.xml | 18 ++
.../addon-services/ONEFS/1.0.0/metainfo.xml | 1 +
.../addon-services/ONEFS/1.0.0/metrics.json | 256 +++++++++++++++----
.../ONEFS/1.0.0/package/scripts/params_linux.py | 2 +-
.../addon-services/ONEFS/1.0.0/widgets.json | 191 +++++++++++++-
9 files changed, 565 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
index f6ae54f..53227e5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
@@ -22,8 +22,11 @@ import static java.util.stream.Collectors.toList;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
/**
*
@@ -61,17 +64,51 @@ public final class JMXMetricHolder {
.collect(toList());
}
- public Optional<Object> find(String property) {
- String propertyName = property.split("/")[0];
- String propertyValue = property.split("/")[1];
+ public Optional<Object> find(String pattern) {
+ JmxPattern jmxPattern = JmxPattern.parse(pattern);
return beans.stream()
- .filter(each -> propertyName.equals(name(each)))
- .map(each -> each.get(propertyValue))
- .filter(Objects::nonNull)
+ .map(jmxPattern::extract)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
.findFirst();
}
- private String name(Map<String, Object> bean) {
- return bean.containsKey(NAME_KEY) ? (String) bean.get(NAME_KEY) : null;
+ private static class JmxPattern {
+ private static final Pattern PATTERN = Pattern.compile("(.*?)\\[(\\S+?)\\]");
+ private final String beanName;
+ private final String propertyName;
+ private final @Nullable String key;
+
+ public static JmxPattern parse(String property) {
+ String beanName = property.split("/")[0];
+ String propertyName = property.split("/")[1];
+ String key = null;
+ Matcher matcher = PATTERN.matcher(propertyName);
+ if (matcher.matches()) {
+ propertyName = matcher.group(1);
+ key = matcher.group(2);
+ }
+ return new JmxPattern(beanName, propertyName, key);
+ }
+
+ private JmxPattern(String beanName, String propertyName, String key) {
+ this.beanName = beanName;
+ this.propertyName = propertyName;
+ this.key = key;
+ }
+
+ public Optional<Object> extract(Map<String, Object> bean) {
+ return beanName.equals(name(bean))
+ ? Optional.ofNullable(lookupByKey(bean.get(propertyName)))
+ : Optional.empty();
+ }
+
+ public Object lookupByKey(Object bean) {
+ return key != null && bean instanceof Map ? ((Map) bean).get(key) : bean;
+ }
+
+ private String name(Map<String, Object> bean) {
+ return bean.containsKey(NAME_KEY) ? (String) bean.get(NAME_KEY) : null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
index a7e11e1..66d09a5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
@@ -17,9 +17,8 @@
*/
package org.apache.ambari.server.state.alert;
-import static org.apache.ambari.server.state.alert.Reporting.ReportingType.PERCENT;
-
import java.text.MessageFormat;
+import java.util.ArrayList;
import java.util.List;
import org.apache.ambari.server.alerts.Threshold;
@@ -208,7 +207,7 @@ public class Reporting {
}
public AlertState state(double value) {
- return getThreshold().state(PERCENT == getType() ? value * 100 : value);
+ return getThreshold().state(value);
}
private Threshold getThreshold() {
@@ -216,7 +215,9 @@ public class Reporting {
}
public String formatMessage(double value, List<Object> args) {
- return MessageFormat.format(message(value), args.toArray());
+ List<Object> copy = new ArrayList<>(args);
+ copy.add(value);
+ return MessageFormat.format(message(value), copy.toArray());
}
private String message(double value) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
index 61d5bae..99eb961 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
@@ -42,6 +42,13 @@ public class JMXMetricHolderTest {
new HashMap<String, Object>() {{
put("name", "bean2");
put("value", "val2");
+ }},
+ new HashMap<String, Object>() {{
+ put("name", "nested");
+ put("value", new HashMap<String, Object>() {{
+ put("key1", "nested-val1");
+ put("key2", "nested-val2");
+ }});
}}
));
}
@@ -58,4 +65,11 @@ public class JMXMetricHolderTest {
List<Object> result = metrics.findAll(asList("bean1/value", "bean2/value", "bean3/notfound"));
assertThat(result, hasItems("val1", "val2"));
}
+
+
+ @Test
+ public void testFindNestedBean() throws Exception {
+ List<Object> result = metrics.findAll(asList("nested/value[key1]", "nested/value[key2]"));
+ assertThat(result, hasItems("nested-val1", "nested-val2"));
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
index 5718721..c84a5be 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/alerts.json
@@ -2,10 +2,10 @@
"ONEFS":{
"service": [
{
- "name": "onefs_namenode_cpu",
- "label": "OneFS NameNode Host CPU Utilization",
- "description": "This host-level alert is triggered if CPU utilization of the NameNode exceeds certain warning and critical thresholds. It checks the NameNode JMX Servlet for the SystemCPULoad property. The threshold values are in percent.",
- "interval": 1,
+ "name": "namenode_onefs_capacity_utilization",
+ "label": "OneFS Capacity Utilization",
+ "description": "This service-level alert is triggered if the OneFS capacity utilization exceeds the configured warning and critical thresholds. It checks the OneFS NameNode for the CapacityUsed and CapacityRemaining properties. The threshold values are in percent.",
+ "interval": 2,
"scope": "HOST",
"enabled": true,
"source": {
@@ -20,26 +20,102 @@
},
"reporting": {
"ok": {
- "text": "{1} CPU, load {0,number,percent}"
+ "text": "Capacity Used:[{2,number,#.##}%, {0}], Capacity Remaining:[{1}]"
},
"warning": {
- "text": "{1} CPU, load {0,number,percent}",
- "value": 5
+ "text": "Capacity Used:[{2,number,#.##}%, {0}], Capacity Remaining:[{1}]",
+ "value": 75
},
"critical": {
- "text": "{1} CPU, load {0,number,percent}",
- "value": 20
+ "text": "Capacity Used:[{2,number,#.##}%, {0}], Capacity Remaining:[{1}]",
+ "value": 80
+ }
+ },
+ "jmx": {
+ "property_list": [
+ "Hadoop:service=NameNode,name=FSNamesystem/CapacityUsed",
+ "Hadoop:service=NameNode,name=FSNamesystem/CapacityRemaining"
+ ],
+ "value": "new Double({0})/({0} + {1}) * 100.0"
+ }
+ }
+ },
+ {
+ "name": "namenode_onefs_blocks_health",
+ "label": "OneFS NameNode Blocks Health",
+ "description": "This service-level alert is triggered if the number of corrupt or missing blocks exceeds the configured critical threshold. The threshold values are in blocks.",
+ "interval": 2,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SERVER",
+ "class": "org.apache.ambari.server.alerts.JmxServerSideAlert",
+ "uri": {
+ "http": "${hdfs-site/dfs.namenode.http-address}",
+ "https": "${hdfs-site/dfs.namenode.https-address}",
+ "https_property": "${hdfs-site/dfs.http.policy}",
+ "https_property_value": "HTTPS_ONLY",
+ "connection_timeout": 5.0
+ },
+ "reporting": {
+ "ok": {
+ "text": "Missing Blocks:[{0}]"
+ },
+ "warning": {
+ "text": "Missing Blocks:[{0}]",
+ "value": 1
+ },
+ "critical": {
+ "text": "Missing Blocks:[{0}]",
+ "value": 1
+ }
+ },
+ "jmx": {
+ "property_list": [
+ "Hadoop:service=NameNode,name=FSNamesystem/MissingBlocks"
+ ],
+ "value": "{0}"
+ }
+ }
+ },
+ {
+ "name": "onefs_heap_usage",
+ "label": "OneFS Heap Usage",
+ "description": "This host-level alert is triggered if heap usage goes past thresholds on the DataNode. The threshold values are in percent.",
+ "interval": 2,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SERVER",
+ "class": "org.apache.ambari.server.alerts.JmxServerSideAlert",
+ "uri": {
+ "http": "${hdfs-site/dfs.namenode.http-address}",
+ "https": "${hdfs-site/dfs.namenode.https-address}",
+ "https_property": "${hdfs-site/dfs.http.policy}",
+ "https_property_value": "HTTPS_ONLY",
+ "connection_timeout": 5.0
+ },
+ "reporting": {
+ "ok": {
+ "text": "Used Heap:[{2,number,#.##}%, {0} B], Max Heap: {1} B"
+ },
+ "warning": {
+ "text": "Used Heap:[{2,number,#.##}%, {0} B], Max Heap: {1} B",
+ "value": 80
},
- "units" : "%",
- "type": "PERCENT"
- },
- "jmx": {
- "property_list": [
- "java.lang:type=OperatingSystem/SystemCpuLoad",
- "java.lang:type=OperatingSystem/AvailableProcessors"
- ]
- }
+ "critical": {
+ "text": "Used Heap:[{2,number,#.##}%, {0} B], Max Heap: {1} B",
+ "value": 90
+ }
+ },
+ "jmx": {
+ "property_list": [
+ "java.lang:type=Memory/HeapMemoryUsage[used]",
+ "java.lang:type=Memory/HeapMemoryUsage[max]"
+ ],
+ "value": "100.0 - (new Double({1} - {0})/{1} * 100.0)"
}
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
index bb671cc..5558f59 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hadoop-env.xml
@@ -193,6 +193,24 @@
<value-attributes>
<type>user</type>
<overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>hadoop-env</type>
+ <name>hdfs_group</name>
+ </property>
+ </user-groups>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hdfs_group</name>
+ <display-name>HDFS User Group</display-name>
+ <value>hdfs_group</value>
+ <property-type>GROUP</property-type>
+ <description>HDFS user group.</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
index f20bcf8..82e2022 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
@@ -27,6 +27,7 @@
<components>
<component>
<name>ONEFS_CLIENT</name>
+ <timelineAppid>NAMENODE</timelineAppid>
<displayName>OneFS Client</displayName>
<category>CLIENT</category>
<componentType>HCFS_CLIENT</componentType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
index cd705eb..335bfba 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
@@ -1,55 +1,211 @@
{
- "ONEFS_CLIENT": {
- "Component": [
+ "ONEFS_CLIENT" : {
+ "Component" : [
{
- "type": "ganglia",
- "metrics": {
- "default": {
- "metrics/onefs/demo/counter": {
- "metric": "onefs.demo.counter",
- "pointInTime": true,
- "temporal": true
- }
- }
- }
- },
- {
- "type": "jmx",
- "properties": {
- "url_config_type": "hdfs-site",
- "url_property_name": "dfs.namenode.http-address"
- },
- "metrics": {
- "default": {
- "metrics/dfs/namenode/Used": {
- "metric": "Hadoop:service=NameNode,name=NameNodeInfo.Used",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemHeapUsedM",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/HeapMemoryMax": {
- "metric": "java.lang:type=Memory.HeapMemoryUsage[max]",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/HeapMemoryUsed": {
- "metric": "java.lang:type=Memory.HeapMemoryUsage[used]",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/NonHeapMemoryMax": {
- "metric": "java.lang:type=Memory.NonHeapMemoryUsage[max]",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/NonHeapMemoryUsed": {
- "metric": "java.lang:type=Memory.NonHeapMemoryUsage[used]",
- "pointInTime": true,
- "temporal": false
+ "type" : "ganglia",
+ "metrics" : {
+ "default" : {
+ "metrics/load/load_five" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "load_five",
+ "amsHostMetric" : true
+ },
+ "metrics/jvm/JvmMetrics/GcCountConcurrentMarkSweep" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "jvm.JvmMetrics.GcCountConcurrentMarkSweep"
+ },
+ "metrics/cpu/cpu_nice" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "cpu_nice",
+ "amsHostMetric" : true
+ },
+ "metrics/rpc/NumOpenConnections" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "rpc.rpc.NumOpenConnections"
+ },
+ "metrics/network/bytes_in" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "bytes_in",
+ "amsHostMetric" : true
+ },
+ "metrics/cpu/cpu_wio" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "cpu_wio",
+ "amsHostMetric" : true
+ },
+ "metrics/process/proc_run" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "proc_run",
+ "amsHostMetric" : true
+ },
+ "metrics/memory/mem_cached" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "mem_cached",
+ "amsHostMetric" : true
+ },
+ "metrics/disk/write_bytes" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "write_bytes",
+ "amsHostMetric" : true
+ },
+ "metrics/network/bytes_out" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "bytes_out",
+ "amsHostMetric" : true
+ },
+ "metrics/cpu/cpu_idle" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "cpu_idle",
+ "amsHostMetric" : true
+ },
+ "metrics/memory/mem_total" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "mem_total",
+ "amsHostMetric" : true
+ },
+ "metrics/jvm/memHeapUsedM" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "jvm.JvmMetrics.MemHeapUsedM",
+ "unit" : "MB"
+ },
+ "metrics/memory/mem_shared" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "mem_shared",
+ "amsHostMetric" : true
+ },
+ "metrics/disk/write_count" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "write_count",
+ "amsHostMetric" : true
+ },
+ "metrics/network/pkts_in" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "pkts_in",
+ "amsHostMetric" : true
+ },
+ "metrics/network/pkts_out" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "pkts_out",
+ "amsHostMetric" : true
+ },
+ "metrics/load/load_one" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "load_one",
+ "amsHostMetric" : true
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "rpc.rpc.RpcProcessingTimeAvgTime"
+ },
+ "metrics/jvm/JvmMetrics/GcTimeMillisConcurrentMarkSweep" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep"
+ },
+ "metrics/memory/swap_total" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "swap_total",
+ "amsHostMetric" : true
+ },
+ "metrics/disk/read_bytes" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "read_bytes",
+ "amsHostMetric" : true
+ },
+ "metrics/jvm/gcTimeMillis" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "jvm.JvmMetrics.GcTimeMillis"
+ },
+ "metrics/jvm/gcCount" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "jvm.JvmMetrics.GcCount"
+ },
+ "metrics/memory/swap_free" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "swap_free",
+ "amsHostMetric" : true
+ },
+ "metrics/process/proc_total" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "proc_total",
+ "amsHostMetric" : true
+ },
+ "metrics/load/load_fifteen" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "load_fifteen",
+ "amsHostMetric" : true
+ },
+ "metrics/cpu/cpu_user" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "cpu_user",
+ "amsHostMetric" : true
+ },
+ "metrics/jvm/memHeapCommittedM" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "jvm.JvmMetrics.MemHeapCommittedM"
+ },
+ "metrics/disk/disk_total" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "disk_total",
+ "amsHostMetric" : true
+ },
+ "metrics/disk/disk_free" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "disk_free",
+ "amsHostMetric" : true
+ },
+ "metrics/disk/read_count" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "read_count",
+ "amsHostMetric" : true
+ },
+ "metrics/memory/mem_free" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "mem_free",
+ "amsHostMetric" : true
+ },
+ "metrics/rpc/RpcQueueTime_avg_time" : {
+ "temporal" : true,
+ "pointInTime" : false,
+ "metric" : "rpc.rpc.RpcQueueTimeAvgTime"
+ },
+ "metrics/cpu/cpu_system" : {
+ "temporal" : true,
+ "pointInTime" : true,
+ "metric" : "cpu_system",
+ "amsHostMetric" : true
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
index 72e0ae9..b823fc5 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
@@ -30,7 +30,7 @@ from resource_management.libraries.functions import stack_select
config = Script.get_config()
hostname = config["hostname"]
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
user_group = config['configurations']['cluster-env']['user_group']
hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
http://git-wip-us.apache.org/repos/asf/ambari/blob/18785175/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
index 23da1d6..dbde93b 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
@@ -1,29 +1,173 @@
{
"layouts": [
{
- "layout_name": "default_onefs_layout",
+ "layout_name": "default_onefs_dashboard",
"display_name": "Standard ONEFS Dashboard",
"section_name": "ONEFS_SUMMARY",
"widgetLayoutInfo": [
{
- "widget_name": "OneFS metrics demo",
- "description": "Test widget",
- "default_section_name": "ONEFS_SUMMARY",
+ "widget_name": "NameNode GC count",
+ "description": "Count of total garbage collections and count of major type garbage collections of the JVM.",
"widget_type": "GRAPH",
"is_visible": true,
- "scope" : "SERVICE",
"metrics": [
{
- "name": "counter",
- "metric_path": "metrics/onefs/demo/counter",
+ "name": "jvm.JvmMetrics.GcCount._rate",
+ "metric_path": "metrics/jvm/gcCount._rate",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLUSTER"
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate",
+ "metric_path": "metrics/jvm/GcCountConcurrentMarkSweep._rate",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "GC total count",
+ "value": "${jvm.JvmMetrics.GcCount._rate}"
+ },
+ {
+ "name": "GC count of type major collection",
+ "value": "${jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NameNode GC time",
+ "description": "Total time taken by major type garbage collections in milliseconds.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate",
+ "metric_path": "metrics/jvm/GcTimeMillisConcurrentMarkSweep._rate",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
}
],
"values": [
{
- "name": "demo value",
- "values" : "${counter}"
+ "name": "GC time in major collection",
+ "value": "${jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate}"
+ }
+ ],
+ "properties": {
+ "display_unit": "ms",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NameNode Heap",
+ "description": "Heap memory committed and Heap memory used with respect to time.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.JvmMetrics.MemHeapCommittedM",
+ "metric_path": "metrics/jvm/memHeapCommittedM",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "jvm.JvmMetrics.MemHeapUsedM",
+ "metric_path": "metrics/jvm/memHeapUsedM",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM heap committed",
+ "value": "${jvm.JvmMetrics.MemHeapCommittedM}"
+ },
+ {
+ "name": "JVM heap used",
+ "value": "${jvm.JvmMetrics.MemHeapUsedM}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NameNode Host Load",
+ "description": "Percentage of CPU and Memory resources being consumed on NameNode host.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "cpu_system",
+ "metric_path": "metrics/cpu/cpu_system",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_user",
+ "metric_path": "metrics/cpu/cpu_user",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_nice",
+ "metric_path": "metrics/cpu/cpu_nice",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_idle",
+ "metric_path": "metrics/cpu/cpu_idle",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_wio",
+ "metric_path": "metrics/cpu/cpu_wio",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "mem_total",
+ "metric_path": "metrics/memory/mem_total",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "mem_free",
+ "metric_path": "metrics/memory/mem_free",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "CPU utilization",
+ "value": "${((cpu_system + cpu_user + cpu_nice)/(cpu_system + cpu_user + cpu_nice + cpu_idle + cpu_wio)) * 100}"
+ },
+ {
+ "name": "Memory utilization",
+ "value": "${((mem_total - mem_free)/mem_total) * 100}"
}
],
"properties": {
@@ -31,8 +175,33 @@
"time_range": "1",
"display_unit": "%"
}
+ },
+ {
+ "widget_name": "Under Replicated Blocks",
+ "description": "Number represents file blocks that does not meet the replication factor criteria. Its indicative of ONEFS bad health.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks",
+ "metric_path": "metrics/dfs/FSNamesystem/UnderReplicatedBlocks",
+ "service_name": "ONEFS",
+ "component_name": "ONEFS_CLIENT",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "Under Replicated Blocks",
+ "value": "${Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0",
+ "error_threshold": "50"
+ }
}
]
}
]
-}
\ No newline at end of file
+}
[09/51] [abbrv] ambari git commit: AMBARI-22661 Storm service check
fails with StormAtlasHook CNF after cluster deploy (dgrinenko)
Posted by am...@apache.org.
AMBARI-22661 Storm service check fails with StormAtlasHook CNF after cluster deploy (dgrinenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ab342d77
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ab342d77
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ab342d77
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: ab342d77fab0baca7baabc23af8a6f86422c0642
Parents: 856d9a5
Author: Dmytro Grinenko <ha...@apache.org>
Authored: Sat Dec 16 19:26:43 2017 +0200
Committer: Dmytro Grinenko <ha...@apache.org>
Committed: Sat Dec 16 19:27:19 2017 +0200
----------------------------------------------------------------------
.../libraries/functions/setup_atlas_hook.py | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab342d77/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
index 367afc8..985eb37 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
@@ -162,14 +162,9 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
import params
stack_root = Script.get_stack_root()
- atlas_home_dir = os.path.join(stack_root, "current", "atlas-server")
-
- # if this is an upgrade/downagrade, then we must link in the correct version
- # which may not be "current", so change the home directory location
- upgrade_type = Script.get_upgrade_type(default("/commandParams/upgrade_type", ""))
- if upgrade_type is not None:
- version_dir_segment = stack_features.get_stack_feature_version(Script.get_config())
- atlas_home_dir = os.path.join(stack_root, version_dir_segment, "atlas")
+ atlas_component_name = "atlas"
+ stack_version = stack_features.get_stack_feature_version(Script.get_config())
+ atlas_home_dir = os.path.join(stack_root, stack_version, atlas_component_name)
# Will only exist if this host contains Atlas Server
atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name)
[25/51] [abbrv] ambari git commit: AMBARI-22677. Addendum - Don't
call pprint.pformat unnecessarily in Ambari agent (amagyar)
Posted by am...@apache.org.
AMBARI-22677. Addendum - Don't call pprint.pformat unnecessarily in Ambari agent (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ea3fa15
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ea3fa15
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ea3fa15
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 9ea3fa154adb1328000bfb8170ea93b7faa66783
Parents: 4e731d3
Author: Attila Magyar <am...@hortonworks.com>
Authored: Wed Dec 20 17:34:45 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Wed Dec 20 17:34:45 2017 +0100
----------------------------------------------------------------------
ambari-agent/src/main/python/ambari_agent/Controller.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ea3fa15/ambari-agent/src/main/python/ambari_agent/Controller.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Controller.py b/ambari-agent/src/main/python/ambari_agent/Controller.py
index ef50440..e4f148f 100644
--- a/ambari-agent/src/main/python/ambari_agent/Controller.py
+++ b/ambari-agent/src/main/python/ambari_agent/Controller.py
@@ -164,14 +164,12 @@ class Controller(threading.Thread):
while not self.isRegistered:
try:
data = json.dumps(self.register.build(self.version))
- prettyData = pprint.pformat(data)
-
try:
server_ip = socket.gethostbyname(self.hostname)
- logger.info("Registering with %s (%s) (agent=%s)", self.hostname, server_ip, prettyData)
+ logger.info("Registering with %s (%s) (agent=%s)", self.hostname, server_ip, data)
except socket.error:
logger.warn("Unable to determine the IP address of '%s', agent registration may fail (agent=%s)",
- self.hostname, prettyData)
+ self.hostname, data)
ret = self.sendRequest(self.registerUrl, data)
[47/51] [abbrv] ambari git commit: AMBARI-22319. Allow the same
config type to belong to multiple services (amagyar)
Posted by am...@apache.org.
AMBARI-22319. Allow the same config type to belong to multiple services (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/80af28ee
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/80af28ee
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/80af28ee
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 80af28ee262a3c0ed7eaeaa5b7c1d54895ce49f6
Parents: 1878517
Author: Attila Magyar <am...@hortonworks.com>
Authored: Mon Oct 30 11:34:27 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../AmbariManagementControllerImpl.java | 8 +-
.../server/controller/internal/Stack.java | 14 ++++
.../ambari/server/state/ConfigHelper.java | 2 +-
.../server/state/cluster/ClusterImpl.java | 79 +++++++-------------
.../ambari/server/topology/AmbariContext.java | 7 +-
.../server/state/cluster/ClusterTest.java | 4 +
.../server/topology/AmbariContextTest.java | 5 +-
7 files changed, 58 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 198b617..a2d8494 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -802,14 +802,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
// If the config type is for a service, then allow a user with SERVICE_MODIFY_CONFIGS to
// update, else ensure the user has CLUSTER_MODIFY_CONFIGS
- String service = null;
+ String service = cluster.getServiceByConfigType(configType);
- try {
- service = cluster.getServiceForConfigTypes(Collections.singleton(configType));
- } catch (IllegalArgumentException e) {
- // Ignore this since we may have hit a config type that spans multiple services. This may
- // happen in unit test cases but should not happen with later versions of stacks.
- }
// Get the changes so that the user's intention can be determined. For example, maybe
// the user wants to change the run-as user for a service or maybe the the cluster-wide
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
index f8feef2..912d9be 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
@@ -18,10 +18,12 @@
package org.apache.ambari.server.controller.internal;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -494,6 +496,18 @@ public class Stack {
"Specified configuration type is not associated with any service: " + config);
}
+ public List<String> getServicesForConfigType(String config) {
+ List<String> serviceNames = new ArrayList<>();
+ for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) {
+ Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue();
+ String serviceName = entry.getKey();
+ if (typeMap.containsKey(config) && !getExcludedConfigurationTypes(serviceName).contains(config)) {
+ serviceNames.add(serviceName);
+ }
+ }
+ return serviceNames;
+ }
+
/**
* Return the dependencies specified for the given component.
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 6813fc0..3b0ee0f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -1123,7 +1123,7 @@ public class ConfigHelper {
if (null != baseConfig) {
try {
- String service = cluster.getServiceForConfigTypes(Collections.singleton(type));
+ String service = cluster.getServiceByConfigType(type);
if (!serviceMapped.containsKey(service)) {
serviceMapped.put(service, new HashSet<>());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 2266c62..2f3afd6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -18,6 +18,8 @@
package org.apache.ambari.server.state.cluster;
+import static java.util.stream.Collectors.toList;
+
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -29,13 +31,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReadWriteLock;
-import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.persistence.EntityManager;
@@ -1135,7 +1137,7 @@ public class ClusterImpl implements Cluster {
return clusterDAO.getLatestConfigurationsWithTypes(clusterId, getDesiredStackVersion(), types)
.stream()
.map(clusterConfigEntity -> configFactory.createExisting(this, clusterConfigEntity))
- .collect(Collectors.toList());
+ .collect(toList());
}
@Override
@@ -1581,37 +1583,35 @@ public class ClusterImpl implements Cluster {
@Override
public String getServiceForConfigTypes(Collection<String> configTypes) {
- //debug
- LOG.info("Looking for service for config types {}", configTypes);
- String serviceName = null;
- for (String configType : configTypes) {
- for (Entry<String, String> entry : serviceConfigTypes.entries()) {
- if (StringUtils.equals(entry.getValue(), configType)) {
- if (serviceName != null) {
- if (entry.getKey()!=null && !StringUtils.equals(serviceName, entry.getKey())) {
- throw new IllegalArgumentException(String.format("Config type %s belongs to %s service, " +
- "but also qualified for %s", configType, serviceName, entry.getKey()));
- }
- } else {
- serviceName = entry.getKey();
- }
- }
- }
+ List<String> serviceNames = configTypes.stream()
+ .map(this::getServiceByConfigType)
+ .filter(Objects::nonNull)
+ .collect(toList());
+ boolean allTheSame = new HashSet<>(serviceNames).size() <= 1;
+ if (!allTheSame) {
+ throw new IllegalArgumentException(String.format(
+ "Config types: %s should belong to a single installed service. But they belong to: %s", configTypes, serviceNames));
}
- LOG.info("Service {} returning", serviceName);
- return serviceName;
+ return serviceNames.isEmpty() ? null : serviceNames.get(0);
+ }
+
+ public List<String> serviceNameByConfigType(String configType) {
+ return serviceConfigTypes.entries().stream()
+ .filter(entry -> StringUtils.equals(entry.getValue(), configType))
+ .map(entry -> entry.getKey())
+ .collect(toList());
}
@Override
public String getServiceByConfigType(String configType) {
- for (Entry<String, String> entry : serviceConfigTypes.entries()) {
- String serviceName = entry.getKey();
- String type = entry.getValue();
- if (StringUtils.equals(type, configType)) {
- return serviceName;
- }
- }
- return null;
+ return serviceNameByConfigType(configType).stream()
+ .filter(this::isServiceInstalled)
+ .findFirst()
+ .orElse(null);
+ }
+
+ private boolean isServiceInstalled(String serviceName) {
+ return services.get(serviceName) != null;
}
@Override
@@ -1894,28 +1894,7 @@ public class ClusterImpl implements Cluster {
@Transactional
ServiceConfigVersionResponse applyConfigs(Set<Config> configs, String user, String serviceConfigVersionNote) {
-
- String serviceName = null;
- for (Config config : configs) {
- for (Entry<String, String> entry : serviceConfigTypes.entries()) {
- if (StringUtils.equals(entry.getValue(), config.getType())) {
- if (serviceName == null) {
- serviceName = entry.getKey();
- break;
- } else if (!serviceName.equals(entry.getKey())) {
- String error = String.format("Updating configs for multiple services by a " +
- "single API request isn't supported. Conflicting services %s and %s for %s",
- serviceName, entry.getKey(), config.getType());
- IllegalArgumentException exception = new IllegalArgumentException(error);
- LOG.error(error + ", config version not created for {}", serviceName);
- throw exception;
- } else {
- break;
- }
- }
- }
- }
-
+ String serviceName = getServiceForConfigTypes(configs.stream().map(Config::getType).collect(toList()));
// update the selected flag for every config type
ClusterEntity clusterEntity = getClusterEntity();
Collection<ClusterConfigEntity> clusterConfigs = clusterEntity.getClusterConfigEntities();
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 933afa2..9e7eb17 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -732,7 +732,12 @@ public class AmbariContext {
// iterate over topo host group configs which were defined in
for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) {
String type = entry.getKey();
- String service = stack.getServiceForConfigType(type);
+ List<String> services = stack.getServicesForConfigType(type);
+ String service = services.stream()
+ .filter(each -> topology.getBlueprint().getServices().contains(each))
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException("Specified configuration type is not associated with any service: " + type));
+
Config config = configFactory.createReadOnly(type, groupName, entry.getValue(), null);
//todo: attributes
Map<String, Config> serviceConfigs = groupConfigs.get(service);
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
index 4b09c6d..8b37f72 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
@@ -1203,6 +1203,7 @@ public class ClusterTest {
@Test
public void testServiceConfigVersions() throws Exception {
createDefaultCluster();
+ c1.addService("HDFS", helper.getOrCreateRepositoryVersion(new StackId("HDP", "0.1"), "0.1"));
Config config1 = configFactory.createNew(c1, "hdfs-site", "version1",
new HashMap<String, String>() {{ put("a", "b"); }}, new HashMap<>());
@@ -1261,6 +1262,7 @@ public class ClusterTest {
@Test
public void testSingleServiceVersionForMultipleConfigs() throws Exception {
createDefaultCluster();
+ c1.addService("HDFS", helper.getOrCreateRepositoryVersion(new StackId("HDP", "0.1"), "0.1"));
Config config1 = configFactory.createNew(c1, "hdfs-site", "version1",
new HashMap<String, String>() {{ put("a", "b"); }}, new HashMap<>());
@@ -1383,6 +1385,7 @@ public class ClusterTest {
public void testAllServiceConfigVersionsWithConfigGroups() throws Exception {
// Given
createDefaultCluster();
+ c1.addService("HDFS", helper.getOrCreateRepositoryVersion(new StackId("HDP", "0.1"), "0.1"));
Config hdfsSiteConfigV1 = configFactory.createNew(c1, "hdfs-site", "version1",
ImmutableMap.of("p1", "v1"), ImmutableMap.of());
@@ -1442,6 +1445,7 @@ public class ClusterTest {
public void testAllServiceConfigVersionsWithDeletedConfigGroups() throws Exception {
// Given
createDefaultCluster();
+ c1.addService("HDFS", helper.getOrCreateRepositoryVersion(new StackId("HDP", "0.1"), "0.1"));
Config hdfsSiteConfigV1 = configFactory.createNew(c1, "hdfs-site", "version1",
ImmutableMap.of("p1", "v1"), ImmutableMap.of());
http://git-wip-us.apache.org/repos/asf/ambari/blob/80af28ee/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
index 0deeae9..5128c65 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
+import static java.util.Collections.singletonList;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMock;
@@ -215,7 +216,7 @@ public class AmbariContextTest {
expect(repositoryVersion.getType()).andReturn(RepositoryType.STANDARD).atLeastOnce();
expect(repositoryVersionDAO.findByStack(EasyMock.anyObject(StackId.class))).andReturn(
- Collections.singletonList(repositoryVersion)).atLeastOnce();
+ singletonList(repositoryVersion)).atLeastOnce();
replay(repositoryVersionDAO, repositoryVersion);
context.configFactory = configFactory;
@@ -240,7 +241,7 @@ public class AmbariContextTest {
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
for (Map.Entry<String, String> entry : configTypeServiceMapping.entrySet()) {
- expect(stack.getServiceForConfigType(entry.getKey())).andReturn(entry.getValue()).anyTimes();
+ expect(stack.getServicesForConfigType(entry.getKey())).andReturn(singletonList(entry.getValue())).anyTimes();
}
expect(controller.getClusters()).andReturn(clusters).anyTimes();
[44/51] [abbrv] ambari git commit: AMBARI-22507. Ambari UI changes to
support Isilon OneFS (amagyar)
Posted by am...@apache.org.
AMBARI-22507. Ambari UI changes to support Isilon OneFS (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9378a5fb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9378a5fb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9378a5fb
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 9378a5fb30d328405e61e9e89ec1ac48a6fa2e54
Parents: 7a982d2
Author: Attila Magyar <am...@hortonworks.com>
Authored: Tue Nov 28 22:05:49 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../app/controllers/wizard/step4_controller.js | 13 ++++-
ambari-web/app/mappers/stack_service_mapper.js | 3 +
ambari-web/app/messages.js | 2 +
ambari-web/app/models/stack_service.js | 17 ++++++
ambari-web/app/templates/wizard/step4.hbs | 58 ++++++++++++++++----
.../test/controllers/wizard/step4_test.js | 29 +++++++++-
6 files changed, 107 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9378a5fb/ambari-web/app/controllers/wizard/step4_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step4_controller.js b/ambari-web/app/controllers/wizard/step4_controller.js
index 8a62f3b..9defa12 100644
--- a/ambari-web/app/controllers/wizard/step4_controller.js
+++ b/ambari-web/app/controllers/wizard/step4_controller.js
@@ -34,11 +34,12 @@ App.WizardStep4Controller = Em.ArrayController.extend({
*/
isAllChecked: function(key, value) {
if (arguments.length > 1) {
- this.filterProperty('isDisabled', false).setEach('isSelected', value);
+ this.filterProperty('isDisabled', false).filterProperty('isDFS', false).setEach('isSelected', value);
return value;
}
return this.filterProperty('isInstalled', false).
filterProperty('isHiddenOnSelectServicePage', false).
+ filterProperty('isDFS', false).
everyProperty('isSelected', true);
}.property('@each.isSelected'),
@@ -59,6 +60,16 @@ App.WizardStep4Controller = Em.ArrayController.extend({
errorStack: [],
/**
+ * Services which are HDFS compatible
+ */
+ fileSystems: function() {
+ var fileSystems = this.filterProperty('isDFS', true);;
+ return fileSystems.map(function(fs) {
+ return App.FileSystem.create({content: fs, services: fileSystems});
+ });
+ }.property('@each.isDFS'),
+
+ /**
* Drop errorStack content on selected state changes.
*/
clearErrors: function() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9378a5fb/ambari-web/app/mappers/stack_service_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/stack_service_mapper.js b/ambari-web/app/mappers/stack_service_mapper.js
index f49b4d0..368a182 100644
--- a/ambari-web/app/mappers/stack_service_mapper.js
+++ b/ambari-web/app/mappers/stack_service_mapper.js
@@ -119,6 +119,9 @@ App.stackServiceMapper = App.QuickDataMapper.create({
stackService.is_installable = false;
stackService.is_selected = false;
}
+ if (stackService.service_type === 'HCFS' && stackService.service_name !== 'HDFS') {
+ stackService.is_selected = false;
+ }
if(stackService.selection === "MANDATORY") {
stackService.is_mandatory = true;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9378a5fb/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index d002aa0..b305196 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -877,6 +877,8 @@ Em.I18n.translations = {
'installer.step4.header':'Choose Services',
'installer.step4.body':'Choose which services you want to install on your cluster.',
+ 'installer.step4.headerFS':'Choose File System',
+ 'installer.step4.bodyFS':'Choose which file system you want to install on your cluster.',
'installer.step4.fsCheck.popup.header':'File System Required',
'installer.step4.fsCheck.popup.body':'You did not select a File System but one is required. We will automatically add {0}. Is this OK?',
'installer.step4.multipleDFS.popup.header':'Multiple File Systems Selected',
http://git-wip-us.apache.org/repos/asf/ambari/blob/9378a5fb/ambari-web/app/models/stack_service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js
index a84816e..1b583c1 100644
--- a/ambari-web/app/models/stack_service.js
+++ b/ambari-web/app/models/stack_service.js
@@ -71,6 +71,23 @@ var MissingDependency = Ember.Object.extend({
})
});
+App.FileSystem = Ember.ObjectProxy.extend({
+ content: null,
+ services: [],
+
+ isSelected: function(key, aBoolean) {
+ if (arguments.length > 1) {
+ this.clearAllSelection();
+ this.get('content').set('isSelected', aBoolean);
+ }
+ return this.get('content.isSelected');
+ }.property('content.isSelected', 'services.@each.isSelected'),
+
+ clearAllSelection: function() {
+ this.get('services').setEach('isSelected', false);
+ }
+});
+
/**
* This model loads all services supported by the stack
* The model maps to the http://hostname:8080/api/v1/stacks/HDP/versions/${versionNumber}/services?fields=StackServices/*,serviceComponents/*
http://git-wip-us.apache.org/repos/asf/ambari/blob/9378a5fb/ambari-web/app/templates/wizard/step4.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/wizard/step4.hbs b/ambari-web/app/templates/wizard/step4.hbs
index 5a08250..b738f69 100644
--- a/ambari-web/app/templates/wizard/step4.hbs
+++ b/ambari-web/app/templates/wizard/step4.hbs
@@ -17,9 +17,8 @@
}}
<div id="step4" class="wizard-content col-md-9">
- <h4 class="step-title">{{t installer.step4.header}}</h4>
- <p class="step-description">{{t installer.step4.body}}</p>
-
+ <h4 class="step-title">{{t installer.step4.headerFS}}</h4>
+ <p class="step-description">{{t installer.step4.bodyFS}}</p>
<div class="panel panel-default">
<div class="panel-body">
<table class="table table-hover">
@@ -28,23 +27,20 @@
<th {{QAAttr "service-name-header"}} class="col-md-3">{{t common.service}}</th>
<th {{QAAttr "service-version-header"}} id="service-version">{{t common.version}}</th>
<th {{QAAttr "service-description-header"}} id="service-description">{{t common.description}}</th>
- <th>
- {{view App.CheckboxView data-qa="toggle-all" disabledBinding="isInstalled" checkedBinding="isAllChecked"}}
- </th>
</tr>
</thead>
<tbody>
- {{#each service in controller}}
- {{#unless isHiddenOnSelectServicePage}}
- <tr {{QAAttr "service-row"}} {{bindAttr class="service.isSelected:active service.isSelected:service-selected"}} {{action toggleCheckBox service target="view"}}>
+ {{#each service in controller.fileSystems}}
+ {{#unless service.isHiddenOnSelectServicePage}}
+ <tr {{QAAttr "service-row"}} {{bindAttr class="service.isSelected:active service.isSelected:service-selected"}} {{action toggleCheckBox service target="view"}} >
<td {{QAAttr "service-name"}}>{{service.displayNameOnSelectServicePage}}</td>
<td {{QAAttr "service-version"}}>{{service.serviceVersionDisplay}}</td>
<td {{QAAttr "service-description"}}>{{{service.comments}}}</td>
<td>
<div class="checkbox">
{{view App.CheckboxView checkboxClassNamesBinding="service.serviceName" data-qa="toggle-service"
- disabledBinding="service.isDisabled"
- checkedBinding="service.isSelected"
+ disabledBinding="service.isDisabled"
+ checkedBinding="service.isSelected"
}}
</div>
</td>
@@ -55,6 +51,46 @@
</table>
</div>
</div>
+
+ <h4 class="step-title">{{t installer.step4.header}}</h4>
+ <p class="step-description">{{t installer.step4.body}}</p>
+
+ <div class="panel panel-default">
+ <div class="panel-body">
+ <table class="table table-hover">
+ <thead>
+ <tr>
+ <th {{QAAttr "service-name-header"}} class="col-md-3">{{t common.service}}</th>
+ <th {{QAAttr "service-version-header"}} id="service-version">{{t common.version}}</th>
+ <th {{QAAttr "service-description-header"}} id="service-description">{{t common.description}}</th>
+ <th>
+ {{view App.CheckboxView data-qa="toggle-all" disabledBinding="isInstalled" checkedBinding="isAllChecked"}}
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#each service in controller}}
+ {{#unless service.isHiddenOnSelectServicePage}}
+ {{#unless service.isDFS}}
+ <tr {{QAAttr "service-row"}} {{bindAttr class="service.isSelected:active service.isSelected:service-selected"}} {{action toggleCheckBox service target="view"}}>
+ <td {{QAAttr "service-name"}}>{{service.displayNameOnSelectServicePage}}</td>
+ <td {{QAAttr "service-version"}}>{{service.serviceVersionDisplay}}</td>
+ <td {{QAAttr "service-description"}}>{{{service.comments}}}</td>
+ <td>
+ <div class="checkbox">
+ {{view App.CheckboxView checkboxClassNamesBinding="service.serviceName" data-qa="toggle-service"
+ disabledBinding="service.isDisabled"
+ checkedBinding="service.isSelected" }}
+ </div>
+ </td>
+ </tr>
+ {{/unless}}
+ {{/unless}}
+ {{/each}}
+ </tbody>
+ </table>
+ </div>
+ </div>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9378a5fb/ambari-web/test/controllers/wizard/step4_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/wizard/step4_test.js b/ambari-web/test/controllers/wizard/step4_test.js
index 1cd4863..e9942e0 100644
--- a/ambari-web/test/controllers/wizard/step4_test.js
+++ b/ambari-web/test/controllers/wizard/step4_test.js
@@ -78,18 +78,41 @@ describe('App.WizardStep4Controller', function () {
});
describe('#isAllChecked', function () {
- it('should return true if all services are selected', function () {
+ it('should return true if all non DFS services are selected', function () {
controller.setEach('isInstalled', false);
- controller.findProperty('serviceName', 'HDFS').set('isSelected', true);
+ controller.findProperty('serviceName', 'YARN').set('isSelected', true);
+ controller.findProperty('serviceName', 'HDFS').set('isSelected', false);
expect(controller.get('isAllChecked')).to.equal(true);
});
it('should return false if at least one service is not selected', function () {
- controller.findProperty('serviceName', 'HDFS').set('isSelected', false);
+ controller.findProperty('serviceName', 'YARN').set('isSelected', false);
expect(controller.get('isAllChecked')).to.equal(false);
});
});
+ describe('#fileSystems', function () {
+ beforeEach(function () {
+ controller.clear();
+ controller.set('content', generateSelectedServicesContent(['HDFS', 'GLUSTERFS', 'YARN']));
+ });
+
+ it('returns only DFS services', function () {
+ expect(controller.get('fileSystems')).to.have.length(2);
+ expect(controller.get('fileSystems').mapProperty('serviceName')).to.contain('GLUSTERFS');
+ expect(controller.get('fileSystems').mapProperty('serviceName')).to.contain('HDFS');
+ });
+
+ it('allows selecting only one DFS at a time', function () {
+ var fileSystems = controller.get('fileSystems');
+ fileSystems[0].set('isSelected', true);
+ expect(fileSystems[0].get('isSelected')).to.equal(true);
+ expect(fileSystems[1].get('isSelected')).to.equal(false);
+ fileSystems[1].set('isSelected', true);
+ expect(fileSystems[0].get('isSelected')).to.equal(false);
+ expect(fileSystems[1].get('isSelected')).to.equal(true);
+ });
+ });
describe('#multipleDFSs()', function () {
it('should return true if HDFS is selected and GLUSTERFS is selected', function () {
controller.set('content', generateSelectedServicesContent(['HDFS', 'GLUSTERFS']));
[41/51] [abbrv] ambari git commit: AMBARI-22236. Expression parser
support for JMXServerSide alerts (amagyar)
Posted by am...@apache.org.
AMBARI-22236. Expression parser support for JMXServerSide alerts (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/92f8ee0f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/92f8ee0f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/92f8ee0f
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 92f8ee0f6d904d272738f6231508f1a16643ae9d
Parents: 68e7dd3
Author: Attila Magyar <am...@hortonworks.com>
Authored: Tue Oct 17 16:15:15 2017 +0200
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:58 2017 +0100
----------------------------------------------------------------------
.../server/alerts/JmxServerSideAlert.java | 42 ++++++------
.../server/state/alert/AlertDefinition.java | 13 ++++
.../ambari/server/state/alert/MetricSource.java | 59 ++++++++++++++++-
.../ambari/server/state/alert/Reporting.java | 16 ++---
.../AlertDefinitionResourceProviderTest.java | 4 +-
.../ambari/server/state/alert/JmxInfoTest.java | 68 ++++++++++++++++++++
6 files changed, 162 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/92f8ee0f/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java b/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
index a4b86f8..09eb0a4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
@@ -23,6 +23,7 @@ import static java.util.Collections.singletonList;
import java.net.URI;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
@@ -35,7 +36,6 @@ import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.alert.AlertDefinition;
import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
import org.apache.ambari.server.state.alert.MetricSource;
-import org.apache.ambari.server.state.alert.Reporting;
import org.apache.ambari.server.state.alert.ServerSource;
import org.apache.ambari.server.state.services.MetricsRetrievalService;
import org.slf4j.Logger;
@@ -64,41 +64,35 @@ public class JmxServerSideAlert extends AlertRunnable {
List<Alert> execute(Cluster cluster, AlertDefinitionEntity entity) throws AmbariException {
AlertDefinition alertDef = definitionFactory.coerce(entity);
ServerSource serverSource = (ServerSource) alertDef.getSource();
- URI jmxUrl = jmxUrl(cluster, serverSource);
- JMXMetricHolder metricHolder = jmxMetric(serverSource, jmxUrl);
- return metricHolder == null
- ? emptyList()
- : alerts(alertDef, serverSource.getJmxInfo(), metricHolder, serverSource.getReporting());
+ return buildAlert(jmxMetric(serverSource, cluster), serverSource.getJmxInfo(), alertDef)
+ .map(alert -> singletonList(alert))
+ .orElse(emptyList());
}
- private URI jmxUrl(Cluster cluster, ServerSource serverSource) throws AmbariException {
- return serverSource.getUri().resolve(config(cluster)).resolve(serverSource.getJmxInfo().getUrlSuffix());
+ public Optional<Alert> buildAlert(Optional<JMXMetricHolder> metricHolder, MetricSource.JmxInfo jmxInfo, AlertDefinition alertDef) {
+ return metricHolder.flatMap(metric -> buildAlert(metric, jmxInfo, alertDef));
}
- private Map<String, Map<String, String>> config(Cluster cluster) throws AmbariException {
- return configHelper.getEffectiveConfigProperties(cluster, configHelper.getEffectiveDesiredTags(cluster, null));
+ private Optional<Alert> buildAlert(JMXMetricHolder metricHolder, MetricSource.JmxInfo jmxInfo, AlertDefinition alertDef) {
+ List<Object> allMetrics = metricHolder.findAll(jmxInfo.getPropertyList());
+ return jmxInfo.eval(metricHolder).map(val -> alertDef.buildAlert(val.doubleValue(), allMetrics));
}
- private JMXMetricHolder jmxMetric(ServerSource serverSource, URI jmxUri) {
+ private Optional<JMXMetricHolder> jmxMetric(ServerSource serverSource, Cluster cluster) throws AmbariException {
+ URI jmxUri = jmxUrl(cluster, serverSource);
URLStreamProvider streamProvider = new URLStreamProvider(
serverSource.getUri().getConnectionTimeoutMsec(),
serverSource.getUri().getReadTimeoutMsec(),
ComponentSSLConfiguration.instance());
metricsRetrievalService.submitRequest(MetricsRetrievalService.MetricSourceType.JMX, streamProvider, jmxUri.toString());
- return metricsRetrievalService.getCachedJMXMetric(jmxUri.toString());
+ return Optional.ofNullable(metricsRetrievalService.getCachedJMXMetric(jmxUri.toString()));
+ }
+
+ private URI jmxUrl(Cluster cluster, ServerSource serverSource) throws AmbariException {
+ return serverSource.getUri().resolve(config(cluster)).resolve(serverSource.getJmxInfo().getUrlSuffix());
}
- private List<Alert> alerts(AlertDefinition alertDef, MetricSource.JmxInfo jmxInfo, JMXMetricHolder jmxMetricHolder, Reporting reporting) throws AmbariException {
- List<Object> metrics = jmxMetricHolder.findAll(jmxInfo.getPropertyList());
- if (metrics.isEmpty()) {
- return emptyList();
- }
- if (metrics.get(0) instanceof Number) {
- Alert alert = reporting.alert(((Number) metrics.get(0)).doubleValue(), metrics, alertDef);
- return singletonList(alert);
- } else {
- LOG.info("Unsupported metrics value: {} when running alert: {}", metrics.get(0), alertDef);
- return emptyList();
- }
+ private Map<String, Map<String, String>> config(Cluster cluster) throws AmbariException {
+ return configHelper.getEffectiveConfigProperties(cluster, configHelper.getEffectiveDesiredTags(cluster, null));
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/92f8ee0f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinition.java
index 665430d..f1f21a4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinition.java
@@ -18,7 +18,10 @@
package org.apache.ambari.server.state.alert;
import java.util.HashSet;
+import java.util.List;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
import org.apache.commons.lang.StringUtils;
import com.google.gson.annotations.SerializedName;
@@ -354,6 +357,16 @@ public class AlertDefinition {
}
/**
+ * Map the incoming value to {@link AlertState} and generate an alert with that state.
+ */
+ public Alert buildAlert(double value, List<Object> args) {
+ Reporting reporting = source.getReporting();
+ Alert alert = new Alert(name, null, serviceName, componentName, null, reporting.state(value));
+ alert.setText(reporting.formatMessage(value, args));
+ return alert;
+ }
+
+ /**
* Gets equality based on name only.
*
* @see #deeplyEquals(Object)
http://git-wip-us.apache.org/repos/asf/ambari/blob/92f8ee0f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
index d7283fe..019f3b1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
@@ -17,8 +17,16 @@
*/
package org.apache.ambari.server.state.alert;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.IntStream.range;
+
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
+
+import org.apache.ambari.server.controller.jmx.JMXMetricHolder;
+import org.springframework.expression.spel.standard.SpelExpressionParser;
+import org.springframework.expression.spel.support.StandardEvaluationContext;
import com.google.gson.annotations.SerializedName;
@@ -127,7 +135,7 @@ public class MetricSource extends Source {
@SerializedName("property_list")
private List<String> propertyList;
- private String value;
+ private String value = "{0}";
@SerializedName("url_suffix")
private String urlSuffix = "/jmx";
@@ -136,8 +144,16 @@ public class MetricSource extends Source {
return propertyList;
}
- public String getValue() {
- return value;
+ public void setPropertyList(List<String> propertyList) {
+ this.propertyList = propertyList;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public Value getValue() {
+ return new Value(value);
}
@Override
@@ -159,5 +175,42 @@ public class MetricSource extends Source {
public String getUrlSuffix() {
return urlSuffix;
}
+
+ public Optional<Number> eval(JMXMetricHolder jmxMetricHolder) {
+ List<Object> metrics = jmxMetricHolder.findAll(propertyList);
+ if (metrics.isEmpty()) {
+ return Optional.empty();
+ } else {
+ Object value = getValue().eval(metrics);
+ return value instanceof Number ? Optional.of((Number)value) : Optional.empty();
+ }
+ }
+ }
+
+ public static class Value {
+ private final String value;
+
+ public Value(String value) {
+ this.value = value;
+ }
+
+ /**
+ * Evaluate an expression like "{0}/({0} + {1}) * 100.0" where each positional argument represent a metrics value.
+ * The value is converted to SpEL syntax:
+ * #var0/(#var0 + #var1) * 100.0
+ * then it is evaluated in the context of the metrics parameters.
+ */
+ public Object eval(List<Object> metrics) {
+ StandardEvaluationContext context = new StandardEvaluationContext();
+ context.setVariables(range(0, metrics.size()).boxed().collect(toMap(i -> "var" + i, metrics::get)));
+ return new SpelExpressionParser()
+ .parseExpression(value.replaceAll("(\\{(\\d+)\\})", "#var$2"))
+ .getValue(context);
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/92f8ee0f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
index 51d074e..a7e11e1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
@@ -23,7 +23,6 @@ import java.text.MessageFormat;
import java.util.List;
import org.apache.ambari.server.alerts.Threshold;
-import org.apache.ambari.server.state.Alert;
import org.apache.ambari.server.state.AlertState;
import com.google.gson.annotations.SerializedName;
@@ -208,16 +207,7 @@ public class Reporting {
return true;
}
- /**
- * Map the incoming value to {@link AlertState} and generate an alert with that state.
- */
- public Alert alert(double value, List<Object> args, AlertDefinition alertDef) {
- Alert alert = new Alert(alertDef.getName(), null, alertDef.getServiceName(), alertDef.getComponentName(), null, state(value));
- alert.setText(MessageFormat.format(message(value), args.toArray()));
- return alert;
- }
-
- private AlertState state(double value) {
+ public AlertState state(double value) {
return getThreshold().state(PERCENT == getType() ? value * 100 : value);
}
@@ -225,6 +215,10 @@ public class Reporting {
return new Threshold(getOk().getValue(), getWarning().getValue(), getCritical().getValue());
}
+ public String formatMessage(double value, List<Object> args) {
+ return MessageFormat.format(message(value), args.toArray());
+ }
+
private String message(double value) {
switch (state(value)) {
case OK:
http://git-wip-us.apache.org/repos/asf/ambari/blob/92f8ee0f/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AlertDefinitionResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AlertDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AlertDefinitionResourceProviderTest.java
index 3ef2c48..e8ad651 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AlertDefinitionResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AlertDefinitionResourceProviderTest.java
@@ -457,7 +457,7 @@ public class AlertDefinitionResourceProviderTest {
// JMX
requestProps.put("AlertDefinition/source/jmx/value",
- source.getJmxInfo().getValue());
+ source.getJmxInfo().getValue().toString());
requestProps.put("AlertDefinition/source/jmx/property_list",
source.getJmxInfo().getPropertyList());
@@ -600,7 +600,7 @@ public class AlertDefinitionResourceProviderTest {
// JMX
requestProps.put("AlertDefinition/source/jmx/value",
- source.getJmxInfo().getValue());
+ source.getJmxInfo().getValue().toString());
requestProps.put("AlertDefinition/source/jmx/property_list",
source.getJmxInfo().getPropertyList());
http://git-wip-us.apache.org/repos/asf/ambari/blob/92f8ee0f/ambari-server/src/test/java/org/apache/ambari/server/state/alert/JmxInfoTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/alert/JmxInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/alert/JmxInfoTest.java
new file mode 100644
index 0000000..308ab4f
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/alert/JmxInfoTest.java
@@ -0,0 +1,68 @@
+package org.apache.ambari.server.state.alert;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.HashMap;
+import java.util.Optional;
+
+import org.apache.ambari.server.controller.jmx.JMXMetricHolder;
+import org.junit.Test;
+
+public class JmxInfoTest {
+ private static final String JMX_PROP_NAME1 = "Hadoop:service=NameNode,name=FSNamesystem/CapacityUsed";
+ private static final String JMX_PROP_NAME2 = "Hadoop:service=NameNode,name=FSNamesystem/CapacityRemaining";
+
+ @Test
+ public void testFindJmxMetricsAndCalculateSimpleValue() throws Exception {
+ MetricSource.JmxInfo jmxInfo = jmxInfoWith("{1}");
+ JMXMetricHolder metrics = metrics(12.5, 3.5);
+ assertThat(jmxInfo.eval(metrics), is(Optional.of(3.5)));
+ }
+
+ @Test
+ public void testFindJmxMetricsAndCalculateComplexValue() throws Exception {
+ MetricSource.JmxInfo jmxInfo = jmxInfoWith("2 * ({0} + {1})");
+ JMXMetricHolder metrics = metrics(12.5, 2.5);
+ assertThat(jmxInfo.eval(metrics), is(Optional.of(30.0)));
+ }
+
+ @Test
+ public void testReturnsEmptyWhenJmxPropertyWasNotFound() throws Exception {
+ MetricSource.JmxInfo jmxInfo = new MetricSource.JmxInfo();
+ jmxInfo.setPropertyList(asList("notfound/notfound"));
+ JMXMetricHolder metrics = metrics(1, 2);
+ assertThat(jmxInfo.eval(metrics), is(Optional.empty()));
+ }
+
+ private MetricSource.JmxInfo jmxInfoWith(String value) {
+ MetricSource.JmxInfo jmxInfo = new MetricSource.JmxInfo();
+ jmxInfo.setValue(value);
+ jmxInfo.setPropertyList(asList(JMX_PROP_NAME1, JMX_PROP_NAME2));
+ return jmxInfo;
+ }
+
+ private JMXMetricHolder metrics(final double jmxValue1, final double jmxValue2) {
+ JMXMetricHolder metrics = new JMXMetricHolder();
+ metrics.setBeans(asList(
+ new HashMap<String,Object>() {{
+ put("name", name(JMX_PROP_NAME1));
+ put(key(JMX_PROP_NAME1), jmxValue1);
+ }},
+ new HashMap<String,Object>() {{
+ put("name", name(JMX_PROP_NAME2));
+ put(key(JMX_PROP_NAME2), jmxValue2);
+ }}
+ ));
+ return metrics;
+ }
+
+ private String name(String jmxProp) {
+ return jmxProp.split("/")[0];
+ }
+
+ private String key(String jmxProp) {
+ return jmxProp.split("/")[1];
+ }
+}
\ No newline at end of file
[43/51] [abbrv] ambari git commit: AMBARI-22339. Metrics for OneFS
mpack (amagyar)
Posted by am...@apache.org.
AMBARI-22339. Metrics for OneFS mpack (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/95c46db0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/95c46db0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/95c46db0
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 95c46db09861f0a9aa978b77da0ab9b5e2559bdd
Parents: 80af28e
Author: Attila Magyar <am...@hortonworks.com>
Authored: Mon Nov 6 10:48:44 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../internal/OverriddenMetricsHostProvider.java | 101 +++++++++++++++++++
.../internal/StackDefinedPropertyProvider.java | 9 +-
.../controller/metrics/MetricHostProvider.java | 9 ++
.../timeline/AMSComponentPropertyProvider.java | 4 +-
.../ambari/server/state/ConfigHelper.java | 5 +
.../server/state/stack/MetricDefinition.java | 9 +-
.../OverriddenMetricsHostProviderTest.java | 95 +++++++++++++++++
.../addon-services/ONEFS/1.0.0/metrics.json | 13 ++-
.../addon-services/ONEFS/1.0.0/widgets.json | 65 +++---------
9 files changed, 254 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java
new file mode 100644
index 0000000..cb59b2f
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.internal;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.metrics.MetricHostProvider;
+import org.apache.ambari.server.controller.metrics.MetricsServiceProvider;
+import org.apache.ambari.server.controller.spi.SystemException;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
+
+/**
+ * I'm a special {@link MetricHostProvider} that can override default component host names.
+ */
+public class OverriddenMetricsHostProvider implements MetricHostProvider {
+ private final Map<String, String> overriddenHosts;
+ private final MetricHostProvider metricHostProvider;
+ private final ConfigHelper configHelper;
+ private final VariableReplacementHelper variableReplacer = new VariableReplacementHelper();
+
+ public OverriddenMetricsHostProvider(Map<String, String> overriddenHosts, MetricHostProvider metricHostProvider, ConfigHelper configHelper) {
+ this.overriddenHosts = overriddenHosts;
+ this.metricHostProvider = metricHostProvider;
+ this.configHelper = configHelper;
+ }
+
+ @Override
+ public Optional<String> getExternalHostName(String clusterName, String componentName) {
+ return getOverriddenHost(componentName).map(host -> replaceVariables(clusterName, host));
+ }
+
+ private Optional<String> getOverriddenHost(String componentName) {
+ return Optional.ofNullable(overriddenHosts.get(componentName));
+ }
+
+ private String replaceVariables(String clusterName, String hostName) {
+ try {
+ return hostName(variableReplacer.replaceVariables(hostName, config(clusterName)));
+ } catch (AmbariException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Map<String, Map<String, String>> config(String clusterName) throws AmbariException {
+ return configHelper.getEffectiveConfigProperties(clusterName, null);
+ }
+
+ private String hostName(String resolvedHost) throws AmbariException {
+ return hasScheme(resolvedHost)
+ ? URI.create(resolvedHost).getHost()
+ : URI.create("any://" + resolvedHost).getHost();
+ }
+
+ private boolean hasScheme(String host) {
+ return host.contains("://");
+ }
+
+ @Override
+ public String getHostName(String clusterName, String componentName) throws SystemException {
+ return metricHostProvider.getHostName(clusterName, componentName);
+ }
+
+ @Override
+ public String getCollectorHostName(String clusterName, MetricsServiceProvider.MetricsService service) throws SystemException {
+ return metricHostProvider.getCollectorHostName(clusterName, service);
+ }
+
+ @Override
+ public String getCollectorPort(String clusterName, MetricsServiceProvider.MetricsService service) throws SystemException {
+ return metricHostProvider.getCollectorPort(clusterName, service);
+ }
+
+ @Override
+ public boolean isCollectorHostLive(String clusterName, MetricsServiceProvider.MetricsService service) throws SystemException {
+ return metricHostProvider.isCollectorHostLive(clusterName, service);
+ }
+
+ @Override
+ public boolean isCollectorComponentLive(String clusterName, MetricsServiceProvider.MetricsService service) throws SystemException {
+ return metricHostProvider.isCollectorComponentLive(clusterName, service);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProvider.java
index 8c2198b..861007b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProvider.java
@@ -47,6 +47,7 @@ import org.apache.ambari.server.controller.utilities.StreamProvider;
import org.apache.ambari.server.security.authorization.AuthorizationException;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.stack.Metric;
@@ -155,6 +156,7 @@ public class StackDefinedPropertyProvider implements PropertyProvider {
Map<String, Map<String, PropertyInfo>> jmxMap = new HashMap<>();
List<PropertyProvider> additional = new ArrayList<>();
+ Map<String, String> overriddenHosts = new HashMap<>();
try {
for (Resource r : resources) {
@@ -183,6 +185,7 @@ public class StackDefinedPropertyProvider implements PropertyProvider {
for (MetricDefinition m : defs) {
if (m.getType().equals("ganglia")) {
gangliaMap.put(componentName, getPropertyInfo(m));
+ m.getOverriddenHosts().ifPresent(host -> overriddenHosts.put(componentName, host));
} else if (m.getType().equals("jmx")) {
jmxMap.put(componentName, getPropertyInfo(m));
} else {
@@ -207,7 +210,7 @@ public class StackDefinedPropertyProvider implements PropertyProvider {
MetricsPropertyProvider.createInstance(type, gangliaMap,
streamProvider, sslConfig,
cacheProvider,
- metricHostProvider,
+ metricHostProvider(overriddenHosts),
metricsServiceProvider, clusterNamePropertyId,
hostNamePropertyId, componentNamePropertyId);
@@ -244,6 +247,10 @@ public class StackDefinedPropertyProvider implements PropertyProvider {
return resources;
}
+ private MetricHostProvider metricHostProvider(Map<String, String> overriddenHosts) {
+ return new OverriddenMetricsHostProvider(overriddenHosts, metricHostProvider, injector.getInstance(ConfigHelper.class));
+ }
+
@Override
public Set<String> checkPropertyIds(Set<String> propertyIds) {
return Collections.emptySet();
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
index 28e5381..86d7114 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
@@ -19,6 +19,8 @@ package org.apache.ambari.server.controller.metrics;
import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService;
+import java.util.Optional;
+
import org.apache.ambari.server.controller.spi.SystemException;
public interface MetricHostProvider {
@@ -76,4 +78,11 @@ public interface MetricHostProvider {
* @throws SystemException if unable to get the status of metrics server component
*/
boolean isCollectorComponentLive(String clusterName, MetricsService service) throws SystemException;
+
+ /**
+ * If a component is installed to a host that is managed outside of Ambari (for example OneFS) then this method will return the external hostname.
+ */
+ default Optional<String> getExternalHostName(String clusterName, String componentName) {
+ return Optional.empty();
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
index 487afde..efc0f4f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java
@@ -42,7 +42,9 @@ public class AMSComponentPropertyProvider extends AMSPropertyProvider {
@Override
protected String getHostName(Resource resource) {
- return null;
+ return hostProvider
+ .getExternalHostName((String) resource.getPropertyValue(clusterNamePropertyId), (String) resource.getPropertyValue(componentNamePropertyId))
+ .orElse(null);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 3b0ee0f..f08ef2f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -298,6 +298,11 @@ public class ConfigHelper {
return properties;
}
+ public Map<String, Map<String, String>> getEffectiveConfigProperties(String clusterName, String hostName) throws AmbariException {
+ Cluster cluster = clusters.getCluster(clusterName);
+ return getEffectiveConfigProperties(cluster, getEffectiveDesiredTags(cluster, hostName));
+ }
+
/**
* Get all config attributes for a cluster given a set of configType to
* versionTags map. This helper method merges all the override tags with a
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java
index 635ae35..033b18b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java
@@ -22,6 +22,7 @@ import static java.util.Map.Entry;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonProperty;
@@ -30,6 +31,7 @@ import org.codehaus.jackson.annotate.JsonProperty;
* Used to represent metrics for a stack component.
*/
public class MetricDefinition {
+ private static final String OVERRIDDEN_HOST_PROP = "overridden_host";
private String type = null;
private Map<String, String> properties = null;
private Map<String, Map<String, Metric>> metrics = null;
@@ -74,5 +76,10 @@ public class MetricDefinition {
return sb.toString();
}
-
+
+ public Optional<String> getOverriddenHosts() {
+ return properties == null
+ ? Optional.empty()
+ : Optional.ofNullable(properties.get(OVERRIDDEN_HOST_PROP));
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProviderTest.java
new file mode 100644
index 0000000..cf4b131
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProviderTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.internal;
+
+import static edu.emory.mathcs.backport.java.util.Collections.emptyMap;
+import static org.easymock.EasyMock.expect;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.ambari.server.controller.metrics.MetricHostProvider;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.easymock.EasyMockRule;
+import org.easymock.EasyMockSupport;
+import org.easymock.Mock;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class OverriddenMetricsHostProviderTest extends EasyMockSupport {
+ private static final String COMPONENT_WITH_OVERRIDDEN_HOST = "component1";
+ private static final String CLUSTER_1 = "cluster1";
+ private static final String COMPONENT_WITHOUT_OVERRIDDEN_HOST = "componentWithoutOverriddenHost";
+ private static final String OVERRIDEN_HOST = "overridenHost1";
+ private static final String COMPONENT_WITH_OVERRIDDEN_HOST_PLACEHOLDER = "${hdfs-site/dfs.namenode.http-address}";
+ private static final String RESOLVED_HOST = "resolved.fqdn";
+ @Rule
+ public EasyMockRule mocks = new EasyMockRule(this);
+ @Mock
+ private MetricHostProvider defaultHostProvider;
+ @Mock
+ private ConfigHelper configHelper;
+ private MetricHostProvider hostProvider;
+
+ @Before
+ public void setUp() throws Exception {
+ hostProvider = new OverriddenMetricsHostProvider(overrideHosts(), defaultHostProvider, configHelper);
+ }
+
+ @Test
+ public void testReturnsDefaultWhenNotOverridden() throws Exception {
+ replayAll();
+ assertThat(hostProvider.getExternalHostName(CLUSTER_1, COMPONENT_WITHOUT_OVERRIDDEN_HOST), is(Optional.empty()));
+ verifyAll();
+ }
+
+ @Test
+ public void testReturnOverriddenHostIfPresent() throws Exception {
+ expect(configHelper.getEffectiveConfigProperties(CLUSTER_1, null)).andReturn(emptyMap()).anyTimes();
+ replayAll();
+ assertThat(hostProvider.getExternalHostName(CLUSTER_1, COMPONENT_WITH_OVERRIDDEN_HOST), is(Optional.of(OVERRIDEN_HOST)));
+ verifyAll();
+ }
+
+ @Test
+ public void testReplacesPlaceholderInOverriddenHost() throws Exception {
+ expect(configHelper.getEffectiveConfigProperties(CLUSTER_1, null)).andReturn(config()).anyTimes();
+ replayAll();
+ assertThat(hostProvider.getExternalHostName(CLUSTER_1, COMPONENT_WITH_OVERRIDDEN_HOST_PLACEHOLDER), is(Optional.of(RESOLVED_HOST)));
+ verifyAll();
+ }
+
+ private Map<String, String> overrideHosts() {
+ return new HashMap<String, String>() {{
+ put(COMPONENT_WITH_OVERRIDDEN_HOST, OVERRIDEN_HOST);
+ put(COMPONENT_WITH_OVERRIDDEN_HOST_PLACEHOLDER, "${hdfs-site/dfs.namenode.http-address}");
+ }};
+ }
+
+ private Map<String, Map<String, String>> config() {
+ return new HashMap<String, Map<String, String>>() {{
+ put("hdfs-site", new HashMap<String, String>() {{
+ put("dfs.namenode.http-address", "http://" + RESOLVED_HOST + ":8080");
+ }});
+ }};
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
index 335bfba..30a9f5c 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metrics.json
@@ -3,6 +3,9 @@
"Component" : [
{
"type" : "ganglia",
+ "properties" : {
+ "overridden_host" : "${core-site/fs.defaultFS}"
+ },
"metrics" : {
"default" : {
"metrics/load/load_five" : {
@@ -11,6 +14,12 @@
"metric" : "load_five",
"amsHostMetric" : true
},
+ "metrics/cpu/cpu_num": {
+ "metric": "cpu_num",
+ "pointInTime": true,
+ "temporal": true,
+ "amsHostMetric" : true
+ },
"metrics/jvm/JvmMetrics/GcCountConcurrentMarkSweep" : {
"temporal" : true,
"pointInTime" : true,
@@ -77,7 +86,7 @@
},
"metrics/jvm/memHeapUsedM" : {
"temporal" : true,
- "pointInTime" : false,
+ "pointInTime" : true,
"metric" : "jvm.JvmMetrics.MemHeapUsedM",
"unit" : "MB"
},
@@ -169,7 +178,7 @@
},
"metrics/jvm/memHeapCommittedM" : {
"temporal" : true,
- "pointInTime" : false,
+ "pointInTime" : true,
"metric" : "jvm.JvmMetrics.MemHeapCommittedM"
},
"metrics/disk/disk_total" : {
http://git-wip-us.apache.org/repos/asf/ambari/blob/95c46db0/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
index dbde93b..fe2c52e 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/widgets.json
@@ -15,15 +15,13 @@
"name": "jvm.JvmMetrics.GcCount._rate",
"metric_path": "metrics/jvm/gcCount._rate",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate",
"metric_path": "metrics/jvm/GcCountConcurrentMarkSweep._rate",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
}
],
"values": [
@@ -49,10 +47,9 @@
"metrics": [
{
"name": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate",
- "metric_path": "metrics/jvm/GcTimeMillisConcurrentMarkSweep._rate",
+ "metric_path": "metrics/jvm/JvmMetrics/GcTimeMillisConcurrentMarkSweep._rate",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
}
],
"values": [
@@ -77,15 +74,13 @@
"name": "jvm.JvmMetrics.MemHeapCommittedM",
"metric_path": "metrics/jvm/memHeapCommittedM",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "jvm.JvmMetrics.MemHeapUsedM",
"metric_path": "metrics/jvm/memHeapUsedM",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
}
],
"values": [
@@ -114,50 +109,43 @@
"name": "cpu_system",
"metric_path": "metrics/cpu/cpu_system",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "cpu_user",
"metric_path": "metrics/cpu/cpu_user",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "cpu_nice",
"metric_path": "metrics/cpu/cpu_nice",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "cpu_idle",
"metric_path": "metrics/cpu/cpu_idle",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "cpu_wio",
"metric_path": "metrics/cpu/cpu_wio",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "mem_total",
"metric_path": "metrics/memory/mem_total",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
},
{
"name": "mem_free",
"metric_path": "metrics/memory/mem_free",
"service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ "component_name": "ONEFS_CLIENT"
}
],
"values": [
@@ -175,33 +163,8 @@
"time_range": "1",
"display_unit": "%"
}
- },
- {
- "widget_name": "Under Replicated Blocks",
- "description": "Number represents file blocks that does not meet the replication factor criteria. Its indicative of ONEFS bad health.",
- "widget_type": "NUMBER",
- "is_visible": true,
- "metrics": [
- {
- "name": "Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks",
- "metric_path": "metrics/dfs/FSNamesystem/UnderReplicatedBlocks",
- "service_name": "ONEFS",
- "component_name": "ONEFS_CLIENT",
- "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
- }
- ],
- "values": [
- {
- "name": "Under Replicated Blocks",
- "value": "${Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks}"
- }
- ],
- "properties": {
- "warning_threshold": "0",
- "error_threshold": "50"
- }
}
]
}
]
-}
+}
\ No newline at end of file
[24/51] [abbrv] ambari git commit: AMBARI-22677. Don't call
pprint.pformat unnecessarily in Ambari agent (amagyar)
Posted by am...@apache.org.
AMBARI-22677. Don't call pprint.pformat unnecessarily in Ambari agent (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4e731d31
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4e731d31
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4e731d31
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 4e731d31f7c59bceab3c91565686384645251546
Parents: f3dc1ca
Author: Attila Magyar <am...@hortonworks.com>
Authored: Wed Dec 20 14:16:36 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Wed Dec 20 14:16:36 2017 +0100
----------------------------------------------------------------------
.../src/main/python/ambari_agent/ActionQueue.py | 21 ++++++++++----------
.../python/ambari_agent/CommandStatusDict.py | 2 +-
.../src/main/python/ambari_agent/Controller.py | 17 ++++++++--------
.../ambari_agent/CustomServiceOrchestrator.py | 4 ++--
.../src/main/python/ambari_agent/DataCleaner.py | 4 ++--
.../src/main/python/ambari_agent/Heartbeat.py | 17 ++++++++--------
.../src/main/python/ambari_agent/LiveStatus.py | 3 +--
.../main/python/ambari_agent/PythonExecutor.py | 15 +++++++-------
.../ambari_agent/PythonReflectiveExecutor.py | 3 ++-
.../main/python/ambari_agent/RecoveryManager.py | 17 ++++++++--------
.../ambari_agent/StatusCommandsExecutor.py | 3 ++-
.../python/ambari_agent/alerts/ams_alert.py | 8 +++++---
.../python/ambari_agent/alerts/metric_alert.py | 3 ++-
.../src/main/python/ambari_agent/security.py | 8 ++++----
14 files changed, 66 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index f1b3a42..da4dc4c 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -121,7 +121,8 @@ class ActionQueue(threading.Thread):
for command in commands:
logger.info("Canceling command with taskId = {tid}".format(tid = str(command['target_task_id'])))
- logger.debug(pprint.pformat(command))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(pprint.pformat(command))
task_id = command['target_task_id']
reason = command['reason']
@@ -200,7 +201,7 @@ class ActionQueue(threading.Thread):
def process_command(self, command):
# make sure we log failures
commandType = command['commandType']
- logger.debug("Took an element of Queue (command type = %s)." % commandType)
+ logger.debug("Took an element of Queue (command type = %s).", commandType)
try:
if commandType in [self.EXECUTION_COMMAND, self.BACKGROUND_EXECUTION_COMMAND, self.AUTO_EXECUTION_COMMAND]:
try:
@@ -211,7 +212,7 @@ class ActionQueue(threading.Thread):
if self.controller.recovery_manager.enabled():
self.controller.recovery_manager.stop_execution_command()
else:
- logger.error("Unrecognized command " + pprint.pformat(command))
+ logger.error("Unrecognized command %s", pprint.pformat(command))
except Exception:
logger.exception("Exception while processing {0} command".format(commandType))
@@ -475,14 +476,14 @@ class ActionQueue(threading.Thread):
self.customServiceOrchestrator
def on_background_command_complete_callback(self, process_condensed_result, handle):
- logger.debug('Start callback: %s' % process_condensed_result)
- logger.debug('The handle is: %s' % handle)
+ logger.debug('Start callback: %s', process_condensed_result)
+ logger.debug('The handle is: %s', handle)
status = self.COMPLETED_STATUS if handle.exitCode == 0 else self.FAILED_STATUS
aborted_postfix = self.customServiceOrchestrator.command_canceled_reason(handle.command['taskId'])
if aborted_postfix:
status = self.FAILED_STATUS
- logger.debug('Set status to: %s , reason = %s' % (status, aborted_postfix))
+ logger.debug('Set status to: %s , reason = %s', status, aborted_postfix)
else:
aborted_postfix = ''
@@ -556,11 +557,9 @@ class ActionQueue(threading.Thread):
result['extra'] = component_extra
- logger.debug("Got live status for component " + component + \
- " of service " + str(service) + \
- " of cluster " + str(cluster))
-
- logger.debug(pprint.pformat(result))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("Got live status for component %s of service %s of cluster %s", component, service, cluster)
+ logger.debug(pprint.pformat(result))
if result is not None:
self.commandStatuses.put_command_status(command, result)
except Exception, err:
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py b/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
index 7a97f3f..8ff5818 100644
--- a/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
+++ b/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
@@ -105,7 +105,7 @@ class CommandStatusDict():
# Component status is useful once, removing it
del self.current_state[key]
elif command ['commandType'] in [ActionQueue.AUTO_EXECUTION_COMMAND]:
- logger.debug("AUTO_EXECUTION_COMMAND task deleted " + str(command['commandId']))
+ logger.debug("AUTO_EXECUTION_COMMAND task deleted %s", command['commandId'])
del self.current_state[key]
pass
result = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/Controller.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Controller.py b/ambari-agent/src/main/python/ambari_agent/Controller.py
index bc923c3..ef50440 100644
--- a/ambari-agent/src/main/python/ambari_agent/Controller.py
+++ b/ambari-agent/src/main/python/ambari_agent/Controller.py
@@ -174,8 +174,9 @@ class Controller(threading.Thread):
self.hostname, prettyData)
ret = self.sendRequest(self.registerUrl, data)
- prettyData = pprint.pformat(ret)
- logger.debug("Registration response is %s", prettyData)
+
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("Registration response is %s", pprint.pformat(ret))
# exitstatus is a code of error which was raised on server side.
# exitstatus = 0 (OK - Default)
@@ -203,7 +204,7 @@ class Controller(threading.Thread):
self.cluster_configuration.update_configurations_from_heartbeat(ret)
self.recovery_manager.update_configuration_from_registration(ret)
self.config.update_configuration_from_registration(ret)
- logger.debug("Updated config:" + str(self.config))
+ logger.debug("Updated config: %s", self.config)
# Start StatusCommandExecutor child process or restart it if already running
# in order to receive up to date agent config.
@@ -557,11 +558,11 @@ class Controller(threading.Thread):
if LiveStatus.SERVICES:
return
- logger.debug("Updating components map of cluster " + cluster_name)
+ logger.debug("Updating components map of cluster %s", cluster_name)
# May throw IOError on server connection error
response = self.sendRequest(self.componentsUrl + cluster_name, None)
- logger.debug("Response from %s was %s", self.serverHostname, str(response))
+ logger.debug("Response from %s was %s", self.serverHostname, response)
services, client_components, server_components = [], [], []
for service, components in response['components'].items():
@@ -578,9 +579,9 @@ class Controller(threading.Thread):
LiveStatus.COMPONENTS = server_components
logger.debug("Components map updated")
- logger.debug("LiveStatus.SERVICES" + str(LiveStatus.SERVICES))
- logger.debug("LiveStatus.CLIENT_COMPONENTS" + str(LiveStatus.CLIENT_COMPONENTS))
- logger.debug("LiveStatus.COMPONENTS" + str(LiveStatus.COMPONENTS))
+ logger.debug("LiveStatus.SERVICES %s", LiveStatus.SERVICES)
+ logger.debug("LiveStatus.CLIENT_COMPONENTS %s", LiveStatus.CLIENT_COMPONENTS)
+ logger.debug("LiveStatus.COMPONENTS %s", LiveStatus.COMPONENTS)
def get_status_commands_executor(self):
return self.statusCommandsExecutor
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 645f824..f92ef1d 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -112,7 +112,7 @@ class CustomServiceOrchestrator():
def map_task_to_process(self, task_id, processId):
with self.commands_in_progress_lock:
- logger.debug('Maps taskId=%s to pid=%s' % (task_id, processId))
+ logger.debug('Maps taskId=%s to pid=%s', task_id, processId)
self.commands_in_progress[task_id] = processId
def cancel_command(self, task_id, reason):
@@ -446,7 +446,7 @@ class CustomServiceOrchestrator():
def command_canceled_reason(self, task_id):
with self.commands_in_progress_lock:
if self.commands_in_progress.has_key(task_id):#Background command do not push in this collection (TODO)
- logger.debug('Pop with taskId %s' % task_id)
+ logger.debug('Pop with taskId %s', task_id)
pid = self.commands_in_progress.pop(task_id)
if not isinstance(pid, int):
reason = pid
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/DataCleaner.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/DataCleaner.py b/ambari-agent/src/main/python/ambari_agent/DataCleaner.py
index 0b66fa7..9c3b2e1 100644
--- a/ambari-agent/src/main/python/ambari_agent/DataCleaner.py
+++ b/ambari-agent/src/main/python/ambari_agent/DataCleaner.py
@@ -68,7 +68,7 @@ class DataCleaner(threading.Thread):
logger.info('Data cleanup thread killed.')
def cleanup(self):
- logger.debug("Cleaning up inside directory " + self.data_dir)
+ logger.debug("Cleaning up inside directory %s", self.data_dir)
now = time.time()
total_size_bytes = 0
file_path_to_timestamp = {}
@@ -82,7 +82,7 @@ class DataCleaner(threading.Thread):
file_age = now - os.path.getmtime(file_path)
if file_age > self.file_max_age:
os.remove(os.path.join(file_path))
- logger.debug('Removed file: ' + file_path)
+ logger.debug('Removed file: %s', file_path)
else:
# Since file wasn't deleted in first pass, consider it for the second one with oldest files first
file_size = os.path.getsize(file_path)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
index d7c0325..11ad652 100644
--- a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
+++ b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
@@ -75,12 +75,12 @@ class Heartbeat:
if int(id) == 0:
componentsMapped = False
- logger.debug("Building Heartbeat: {responseId = %s, timestamp = %s, "
- "commandsInProgress = %s, componentsMapped = %s,"
- "recoveryTimestamp = %s}",
- str(id), str(timestamp), repr(commandsInProgress), repr(componentsMapped), str(recovery_timestamp))
-
- logger.debug("Heartbeat: %s", pformat(heartbeat))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("Building Heartbeat: {responseId = %s, timestamp = %s, "
+ "commandsInProgress = %s, componentsMapped = %s, "
+ "recoveryTimestamp = %s}",
+ id, timestamp, commandsInProgress, componentsMapped, recovery_timestamp)
+ logger.debug("Heartbeat: %s", pformat(heartbeat))
hostInfo = HostInfo(self.config)
if add_state:
@@ -93,8 +93,9 @@ class Heartbeat:
mounts = Hardware(config=self.config, cache_info=False).osdisks()
heartbeat['mounts'] = mounts
- logger.debug("agentEnv: %s", str(nodeInfo))
- logger.debug("mounts: %s", str(mounts))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("agentEnv: %s", nodeInfo)
+ logger.debug("mounts: %s", mounts)
if self.collector is not None:
heartbeat['alerts'] = self.collector.alerts()
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
index b6d54ed..a506e28 100644
--- a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
+++ b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
@@ -62,6 +62,5 @@ class LiveStatus:
if active_config is not None:
livestatus['configurationTags'] = active_config
- logger.debug("The live status for component " + str(self.component) +
- " of service " + str(self.service) + " is " + str(livestatus))
+ logger.debug("The live status for component %s of service %s is %s", self.component, self.service, livestatus)
return livestatus
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
index ea6f895..139c141 100644
--- a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
@@ -93,7 +93,8 @@ class PythonExecutor(object):
The structured out file, however, is preserved during multiple invocations that use the same file.
"""
pythonCommand = self.python_command(script, script_params)
- logger.debug("Running command " + pprint.pformat(pythonCommand))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("Running command %s", pprint.pformat(pythonCommand))
if handle is None:
tmpout, tmperr = self.open_subprocess_files(tmpoutfile, tmperrfile, override_output_files, backup_log_files)
@@ -127,7 +128,7 @@ class PythonExecutor(object):
"""
Log some useful information after task failure.
"""
- logger.info("Command " + pprint.pformat(pythonCommand) + " failed with exitcode=" + str(result['exitcode']))
+ logger.info("Command %s failed with exitcode=%s", pprint.pformat(pythonCommand), result['exitcode'])
log_process_information(logger)
def prepare_process_result(self, returncode, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=None):
@@ -138,7 +139,7 @@ class PythonExecutor(object):
(" after waiting %s secs" % str(timeout) if timeout else "")
returncode = 999
result = self.condenseOutput(out, error, returncode, structured_out)
- logger.debug("Result: %s" % result)
+ logger.debug("Result: %s", result)
return result
def read_result_from_files(self, out_path, err_path, structured_out_path):
@@ -223,10 +224,10 @@ class BackgroundThread(threading.Thread):
def run(self):
process_out, process_err = self.pythonExecutor.open_subprocess_files(self.holder.out_file, self.holder.err_file, True)
- logger.debug("Starting process command %s" % self.holder.command)
+ logger.debug("Starting process command %s", self.holder.command)
process = self.pythonExecutor.launch_python_subprocess(self.holder.command, process_out, process_err)
- logger.debug("Process has been started. Pid = %s" % process.pid)
+ logger.debug("Process has been started. Pid = %s", process.pid)
self.holder.handle.pid = process.pid
self.holder.handle.status = BackgroundCommandExecutionHandle.RUNNING_STATUS
@@ -236,6 +237,6 @@ class BackgroundThread(threading.Thread):
self.holder.handle.exitCode = process.returncode
process_condensed_result = self.pythonExecutor.prepare_process_result(process.returncode, self.holder.out_file, self.holder.err_file, self.holder.structured_out_file)
- logger.debug("Calling callback with args %s" % process_condensed_result)
+ logger.debug("Calling callback with args %s", process_condensed_result)
self.holder.handle.on_background_command_complete_callback(process_condensed_result, self.holder.handle)
- logger.debug("Exiting from thread for holder pid %s" % self.holder.handle.pid)
+ logger.debug("Exiting from thread for holder pid %s", self.holder.handle.pid)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py b/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
index b27d7d1..b021320 100644
--- a/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
@@ -46,7 +46,8 @@ class PythonReflectiveExecutor(PythonExecutor):
override_output_files = True, backup_log_files = True,
handle = None, log_info_on_failure=True):
pythonCommand = self.python_command(script, script_params)
- logger.debug("Running command reflectively " + pprint.pformat(pythonCommand))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("Running command reflectively %s", pprint.pformat(pythonCommand))
script_dir = os.path.dirname(script)
self.open_subprocess_files(tmpoutfile, tmperrfile, override_output_files, backup_log_files)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py b/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py
index be335f2..c196fd8 100644
--- a/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py
+++ b/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py
@@ -128,7 +128,7 @@ class RecoveryManager:
def set_paused(self, paused):
if self.paused != paused:
- logger.debug("RecoveryManager is transitioning from isPaused = " + str(self.paused) + " to " + str(paused))
+ logger.debug("RecoveryManager is transitioning from isPaused = %s to %s", self.paused, paused)
self.paused = paused
def enabled(self):
@@ -574,7 +574,8 @@ class RecoveryManager:
if reg_resp and "recoveryConfig" in reg_resp:
- logger.info("RecoverConfig = " + pprint.pformat(reg_resp["recoveryConfig"]))
+ if logger.isEnabledFor(logging.INFO):
+ logger.info("RecoverConfig = %s", pprint.pformat(reg_resp["recoveryConfig"]))
config = reg_resp["recoveryConfig"]
if "type" in config:
if config["type"] in ["AUTO_INSTALL_START", "AUTO_START", "FULL"]:
@@ -691,8 +692,8 @@ class RecoveryManager:
if commands and len(commands) > 0:
for command in commands:
self.store_or_update_command(command)
- if self.EXECUTION_COMMAND_DETAILS in command:
- logger.debug("Details to construct exec commands: " + pprint.pformat(command[self.EXECUTION_COMMAND_DETAILS]))
+ if logger.isEnabledFor(logging.DEBUG) and self.EXECUTION_COMMAND_DETAILS in command:
+ logger.debug("Details to construct exec commands: %s", pprint.pformat(command[self.EXECUTION_COMMAND_DETAILS]))
pass
@@ -748,7 +749,7 @@ class RecoveryManager:
# Store the execution command details
self.remove_command(component)
self.add_command(component, command[self.EXECUTION_COMMAND_DETAILS])
- logger.debug("Stored command details for " + component)
+ logger.debug("Stored command details for %s", component)
else:
logger.warn("Expected field " + self.EXECUTION_COMMAND_DETAILS + " unavailable.")
pass
@@ -855,7 +856,7 @@ class RecoveryManager:
insert_time = self.stored_exec_commands[component_update_key]
age = self._now_() - insert_time
if self.COMMAND_REFRESH_DELAY_SEC < age:
- logger.debug("Removing stored command for component : " + str(component) + " as its " + str(age) + " sec old")
+ logger.debug("Removing stored command for component : %s as it's %s sec old", component, age)
self.remove_command(component)
pass
@@ -867,7 +868,7 @@ class RecoveryManager:
component_update_key = self.COMPONENT_UPDATE_KEY_FORMAT.format(component)
del self.stored_exec_commands[component]
del self.stored_exec_commands[component_update_key]
- logger.debug("Removed stored command for component : " + str(component))
+ logger.debug("Removed stored command for component : %s", component)
return True
finally:
self.__status_lock.release()
@@ -880,7 +881,7 @@ class RecoveryManager:
component_update_key = self.COMPONENT_UPDATE_KEY_FORMAT.format(component)
self.stored_exec_commands[component] = command
self.stored_exec_commands[component_update_key] = self._now_()
- logger.debug("Added command for component : " + str(component))
+ logger.debug("Added command for component : %s", component)
finally:
self.__status_lock.release()
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py b/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
index f42e134..ad90b83 100644
--- a/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
@@ -64,7 +64,8 @@ class SingleProcessStatusCommandsExecutor(StatusCommandsExecutor):
command['serviceName'] + " of cluster " + \
command['clusterName'] + " to the queue.")
self.statusCommandQueue.put(command)
- logger.debug(pprint.pformat(command))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(pprint.pformat(command))
def process_results(self):
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
index 70f100f..29b885b 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
@@ -64,8 +64,9 @@ class AmsAlert(MetricAlert):
# use the URI lookup keys to get a final URI value to query
alert_uri = self._get_uri_from_structure(self.uri_property_keys)
- logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
- self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
+ self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
host = BaseAlert.get_host_from_url(alert_uri.uri)
if host is None:
@@ -94,7 +95,8 @@ class AmsAlert(MetricAlert):
collect_result = self._get_result(value_list[0] if compute_result is None else compute_result)
- logger.debug("[Alert][{0}] Computed result = {1}".format(self.get_name(), str(value_list)))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("[Alert][{0}] Computed result = {1}".format(self.get_name(), str(value_list)))
return (collect_result, value_list)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index 803bdc6..66a1d05 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -107,7 +107,8 @@ class MetricAlert(BaseAlert):
collect_result = self._get_result(value_list[0] if check_value is None else check_value)
- logger.debug("[Alert][{0}] Resolved values = {1}".format(self.get_name(), str(value_list)))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("[Alert][{0}] Resolved values = {1}".format(self.get_name(), str(value_list)))
return (collect_result, value_list)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/security.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/security.py b/ambari-agent/src/main/python/ambari_agent/security.py
index 45de7bb..d752ee6 100644
--- a/ambari-agent/src/main/python/ambari_agent/security.py
+++ b/ambari-agent/src/main/python/ambari_agent/security.py
@@ -48,8 +48,7 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
def connect(self):
self.two_way_ssl_required = self.config.isTwoWaySSLConnection(self.host)
- logger.debug("Server two-way SSL authentication required: %s" % str(
- self.two_way_ssl_required))
+ logger.debug("Server two-way SSL authentication required: %s", self.two_way_ssl_required)
if self.two_way_ssl_required is True:
logger.info(
'Server require two-way SSL authentication. Use it instead of one-way...')
@@ -230,9 +229,10 @@ class CertificateManager():
f.close()
try:
data = json.loads(response)
- logger.debug("Sign response from Server: \n" + pprint.pformat(data))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug("Sign response from Server: \n" + pprint.pformat(data))
except Exception:
- logger.warn("Malformed response! data: " + str(data))
+ logger.warn("Malformed response! data: %s", data)
data = {'result': 'ERROR'}
result = data['result']
if result == 'OK':
[28/51] [abbrv] ambari git commit: AMBARI-22530. Refactor internal
code of handling info between kerberos wizard actions (echekanskiy)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
index ca78dbb..94a6a49 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
@@ -38,10 +38,11 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.RootComponent;
+import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType;
@@ -96,9 +97,6 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
private KerberosKeytabDAO kerberosKeytabDAO;
@Inject
- KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
-
- @Inject
KerberosPrincipalDAO kerberosPrincipalDAO;
@Override
@@ -376,11 +374,11 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
// component.
String componentName = KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME.equals(identity.getName())
? "AMBARI_SERVER_SELF"
- : "AMBARI_SERVER";
+ : RootComponent.AMBARI_SERVER.name();
List<KerberosIdentityDescriptor> componentIdentities = Collections.singletonList(identity);
kerberosHelper.addIdentities(null, componentIdentities,
- null, KerberosHelper.AMBARI_SERVER_HOST_NAME, ambariServerHostID(), "AMBARI", componentName, kerberosConfigurations, currentConfigurations,
+ null, KerberosHelper.AMBARI_SERVER_HOST_NAME, ambariServerHostID(), RootService.AMBARI.name(), componentName, kerberosConfigurations, currentConfigurations,
resolvedKeytabs, realm);
propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
}
@@ -392,7 +390,7 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
// create database records for keytabs that must be presented on cluster
for (ResolvedKerberosKeytab keytab : resolvedKeytabs.values()) {
- kerberosHelper.processResolvedKeytab(keytab);
+ kerberosHelper.createResolvedKeytab(keytab);
}
} catch (IOException e) {
throw new AmbariException(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 5ac1ac3..385a276 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@ -46,7 +46,7 @@ import org.apache.ambari.server.orm.dao.HostConfigMappingDAO;
import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.dao.HostStateDAO;
import org.apache.ambari.server.orm.dao.HostVersionDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.RequestOperationLevelDAO;
import org.apache.ambari.server.orm.dao.ResourceTypeDAO;
import org.apache.ambari.server.orm.dao.ServiceConfigDAO;
@@ -112,8 +112,6 @@ public class ClustersImpl implements Clusters {
@Inject
private RequestOperationLevelDAO requestOperationLevelDAO;
@Inject
- private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
- @Inject
private HostConfigMappingDAO hostConfigMappingDAO;
@Inject
private ServiceConfigDAO serviceConfigDAO;
@@ -129,6 +127,8 @@ public class ClustersImpl implements Clusters {
private TopologyHostInfoDAO topologyHostInfoDAO;
@Inject
private TopologyManager topologyManager;
+ @Inject
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
/**
* Data access object for stacks.
@@ -633,7 +633,7 @@ public class ClustersImpl implements Clusters {
deleteConfigGroupHostMapping(hostEntity.getHostId());
// Remove mapping of principals to the unmapped host
- kerberosPrincipalHostDAO.removeByHost(hostEntity.getHostId());
+ kerberosKeytabPrincipalDAO.removeByHost(hostEntity.getHostId());
}
@Transactional
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
index 7045240..0bd2195 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
@@ -913,21 +913,35 @@ CREATE TABLE kerberos_principal (
CREATE TABLE kerberos_keytab (
keytab_path VARCHAR(255) NOT NULL,
- CONSTRAINT PK_krb_keytab_path_host_id PRIMARY KEY (keytab_path)
+ owner_name VARCHAR(255),
+ owner_access VARCHAR(255),
+ group_name VARCHAR(255),
+ group_access VARCHAR(255),
+ is_ambari_keytab SMALLINT NOT NULL DEFAULT 0,
+ write_ambari_jaas SMALLINT NOT NULL DEFAULT 0,
+ CONSTRAINT PK_kerberos_keytab PRIMARY KEY (keytab_path)
);
-
-CREATE TABLE kerberos_principal_host (
- principal_name VARCHAR(255) NOT NULL,
+CREATE TABLE kerberos_keytab_principal (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
keytab_path VARCHAR(255) NOT NULL,
+ principal_name VARCHAR(255) NOT NULL,
+ host_id BIGINT,
is_distributed SMALLINT NOT NULL DEFAULT 0,
- host_id BIGINT NOT NULL,
- CONSTRAINT PK_kerberos_principal_host PRIMARY KEY (principal_name, keytab_path, host_id),
- CONSTRAINT FK_krb_pr_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
- CONSTRAINT FK_krb_pr_host_principalname FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
- CONSTRAINT FK_krb_pr_host_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path)
+ CONSTRAINT PK_kkp PRIMARY KEY (kkp_id),
+ CONSTRAINT FK_kkp_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path),
+ CONSTRAINT FK_kkp_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
+ CONSTRAINT FK_kkp_principal_name FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
+ CONSTRAINT UNI_kkp UNIQUE(keytab_path, principal_name, host_id)
);
+CREATE TABLE kkp_mapping_service (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
+ service_name VARCHAR(255) NOT NULL,
+ component_name VARCHAR(255) NOT NULL,
+ CONSTRAINT PK_kkp_mapping_service PRIMARY KEY (kkp_id, service_name, component_name),
+ CONSTRAINT FK_kkp_service_principal FOREIGN KEY (kkp_id) REFERENCES kerberos_keytab_principal (kkp_id)
+);
CREATE TABLE kerberos_descriptor
(
@@ -1060,6 +1074,8 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
-- In order for the first ID to be 1, must initialize the ambari_sequences table with a sequence_value of 0.
-- BEGIN;
INSERT INTO ambari_sequences (sequence_name, sequence_value)
+ SELECT 'kkp_id_seq', 0 FROM SYSIBM.SYSDUMMY1
+ UNION ALL
SELECT 'cluster_id_seq', 1 FROM SYSIBM.SYSDUMMY1
UNION ALL
SELECT 'host_id_seq', 0 FROM SYSIBM.SYSDUMMY1
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index c950c7e..23a8fb7 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -931,18 +931,34 @@ CREATE TABLE kerberos_principal (
CREATE TABLE kerberos_keytab (
keytab_path VARCHAR(255) NOT NULL,
- CONSTRAINT PK_krb_keytab_path_host_id PRIMARY KEY (keytab_path)
+ owner_name VARCHAR(255),
+ owner_access VARCHAR(255),
+ group_name VARCHAR(255),
+ group_access VARCHAR(255),
+ is_ambari_keytab SMALLINT NOT NULL DEFAULT 0,
+ write_ambari_jaas SMALLINT NOT NULL DEFAULT 0,
+ CONSTRAINT PK_kerberos_keytab PRIMARY KEY (keytab_path)
);
-CREATE TABLE kerberos_principal_host (
- principal_name VARCHAR(255) NOT NULL,
+CREATE TABLE kerberos_keytab_principal (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
keytab_path VARCHAR(255) NOT NULL,
+ principal_name VARCHAR(255) NOT NULL,
+ host_id BIGINT,
is_distributed SMALLINT NOT NULL DEFAULT 0,
- host_id BIGINT NOT NULL,
- CONSTRAINT PK_kerberos_principal_host PRIMARY KEY (principal_name, keytab_path, host_id),
- CONSTRAINT FK_krb_pr_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
- CONSTRAINT FK_krb_pr_host_principalname FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
- CONSTRAINT FK_krb_pr_host_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path)
+ CONSTRAINT PK_kkp PRIMARY KEY (kkp_id),
+ CONSTRAINT FK_kkp_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path),
+ CONSTRAINT FK_kkp_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
+ CONSTRAINT FK_kkp_principal_name FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
+ CONSTRAINT UNI_kkp UNIQUE(keytab_path, principal_name, host_id)
+);
+
+CREATE TABLE kkp_mapping_service (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
+ service_name VARCHAR(255) NOT NULL,
+ component_name VARCHAR(255) NOT NULL,
+ CONSTRAINT PK_kkp_mapping_service PRIMARY KEY (kkp_id, service_name, component_name),
+ CONSTRAINT FK_kkp_service_principal FOREIGN KEY (kkp_id) REFERENCES kerberos_keytab_principal (kkp_id)
);
CREATE TABLE kerberos_descriptor
@@ -1074,6 +1090,7 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
-- In order for the first ID to be 1, must initialize the ambari_sequences table with a sequence_value of 0.
INSERT INTO ambari_sequences(sequence_name, sequence_value) VALUES
+ ('kkp_id_seq', 0),
('cluster_id_seq', 1),
('host_id_seq', 0),
('host_role_command_id_seq', 1),
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
index 537ae19..0f93c43 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
@@ -910,18 +910,34 @@ CREATE TABLE kerberos_principal (
CREATE TABLE kerberos_keytab (
keytab_path VARCHAR2(255) NOT NULL,
- CONSTRAINT PK_krb_keytab_path_host_id PRIMARY KEY (keytab_path)
+ owner_name VARCHAR2(255),
+ owner_access VARCHAR2(255),
+ group_name VARCHAR2(255),
+ group_access VARCHAR2(255),
+ is_ambari_keytab NUMBER(1) DEFAULT 0 NOT NULL,
+ write_ambari_jaas NUMBER(1) DEFAULT 0 NOT NULL,
+ CONSTRAINT PK_kerberos_keytab PRIMARY KEY (keytab_path)
);
-CREATE TABLE kerberos_principal_host (
- principal_name VARCHAR2(255) NOT NULL,
+CREATE TABLE kerberos_keytab_principal (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
keytab_path VARCHAR2(255) NOT NULL,
- is_distributed NUMBER(1) DEFAULT 0 NOT NULL,
- host_id NUMBER(19) NOT NULL,
- CONSTRAINT PK_kerberos_principal_host PRIMARY KEY (principal_name, keytab_path, host_id),
- CONSTRAINT FK_krb_pr_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
- CONSTRAINT FK_krb_pr_host_principalname FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
- CONSTRAINT FK_krb_pr_host_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path)
+ principal_name VARCHAR2(255) NOT NULL,
+ host_id NUMBER(19),
+ is_distributed NUMBER(1) NOT NULL DEFAULT 0,
+ CONSTRAINT PK_kkp PRIMARY KEY (kkp_id),
+ CONSTRAINT FK_kkp_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path),
+ CONSTRAINT FK_kkp_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
+ CONSTRAINT FK_kkp_principal_name FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
+ CONSTRAINT UNI_kkp UNIQUE(keytab_path, principal_name, host_id)
+);
+
+CREATE TABLE kkp_mapping_service (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
+ service_name VARCHAR(255) NOT NULL,
+ component_name VARCHAR(255) NOT NULL,
+ CONSTRAINT PK_kkp_mapping_service PRIMARY KEY (kkp_id, service_name, component_name),
+ CONSTRAINT FK_kkp_service_principal FOREIGN KEY (kkp_id) REFERENCES kerberos_keytab_principal (kkp_id)
);
CREATE TABLE kerberos_descriptor
@@ -1052,6 +1068,7 @@ CREATE INDEX idx_alert_group_name on alert_group(group_name);
CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
---------inserting some data-----------
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('kkp_id_seq', 0);
-- In order for the first ID to be 1, must initialize the ambari_sequences table with a sequence_value of 0.
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('host_role_command_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('user_id_seq', 1);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index b4952c2..16e978b 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -913,17 +913,35 @@ CREATE TABLE kerberos_principal (
CREATE TABLE kerberos_keytab (
keytab_path VARCHAR(255) NOT NULL,
- CONSTRAINT PK_krb_keytab_path_host_id PRIMARY KEY (keytab_path));
+ owner_name VARCHAR(255),
+ owner_access VARCHAR(255),
+ group_name VARCHAR(255),
+ group_access VARCHAR(255),
+ is_ambari_keytab SMALLINT NOT NULL DEFAULT 0,
+ write_ambari_jaas SMALLINT NOT NULL DEFAULT 0,
+ CONSTRAINT PK_kerberos_keytab PRIMARY KEY (keytab_path));
-CREATE TABLE kerberos_principal_host (
- principal_name VARCHAR(255) NOT NULL,
+
+CREATE TABLE kerberos_keytab_principal (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
keytab_path VARCHAR(255) NOT NULL,
+ principal_name VARCHAR(255) NOT NULL,
+ host_id BIGINT,
is_distributed SMALLINT NOT NULL DEFAULT 0,
- host_id BIGINT NOT NULL,
- CONSTRAINT PK_kerberos_principal_host PRIMARY KEY (principal_name, keytab_path, host_id),
- CONSTRAINT FK_krb_pr_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
- CONSTRAINT FK_krb_pr_host_principalname FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
- CONSTRAINT FK_krb_pr_host_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path));
+ CONSTRAINT PK_kkp PRIMARY KEY (kkp_id),
+ CONSTRAINT FK_kkp_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path),
+ CONSTRAINT FK_kkp_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
+ CONSTRAINT FK_kkp_principal_name FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
+ CONSTRAINT UNI_kkp UNIQUE(keytab_path, principal_name, host_id)
+);
+
+CREATE TABLE kkp_mapping_service (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
+ service_name VARCHAR(255) NOT NULL,
+ component_name VARCHAR(255) NOT NULL,
+ CONSTRAINT PK_kkp_mapping_service PRIMARY KEY (kkp_id, service_name, component_name),
+ CONSTRAINT FK_kkp_service_principal FOREIGN KEY (kkp_id) REFERENCES kerberos_keytab_principal (kkp_id)
+);
CREATE TABLE kerberos_descriptor(
kerberos_descriptor_name VARCHAR(255) NOT NULL,
@@ -1054,6 +1072,7 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
-- In order for the first ID to be 1, must initialize the ambari_sequences table with a sequence_value of 0.
BEGIN;
INSERT INTO ambari_sequences (sequence_name, sequence_value) VALUES
+ ('kkp_id_seq', 0),
('cluster_id_seq', 1),
('host_id_seq', 0),
('user_id_seq', 2),
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
index 4fb0d09..f063638 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
@@ -909,18 +909,34 @@ CREATE TABLE kerberos_principal (
CREATE TABLE kerberos_keytab (
keytab_path VARCHAR(255) NOT NULL,
- CONSTRAINT PK_krb_keytab_path_host_id PRIMARY KEY (keytab_path)
+ owner_name VARCHAR(255),
+ owner_access VARCHAR(255),
+ group_name VARCHAR(255),
+ group_access VARCHAR(255),
+ is_ambari_keytab SMALLINT NOT NULL DEFAULT 0,
+ write_ambari_jaas SMALLINT NOT NULL DEFAULT 0,
+ CONSTRAINT PK_kerberos_keytab PRIMARY KEY (keytab_path)
);
-CREATE TABLE kerberos_principal_host (
- principal_name VARCHAR(255) NOT NULL,
+CREATE TABLE kerberos_keytab_principal (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
keytab_path VARCHAR(255) NOT NULL,
+ principal_name VARCHAR(255) NOT NULL,
+ host_id BIGINT,
is_distributed SMALLINT NOT NULL DEFAULT 0,
- host_id NUMERIC(19) NOT NULL,
- CONSTRAINT PK_kerberos_principal_host PRIMARY KEY (principal_name, keytab_path, host_id),
- CONSTRAINT FK_krb_pr_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
- CONSTRAINT FK_krb_pr_host_principalname FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
- CONSTRAINT FK_krb_pr_host_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path)
+ CONSTRAINT PK_kkp PRIMARY KEY (kkp_id),
+ CONSTRAINT FK_kkp_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path),
+ CONSTRAINT FK_kkp_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
+ CONSTRAINT FK_kkp_principal_name FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
+ CONSTRAINT UNI_kkp UNIQUE(keytab_path, principal_name, host_id)
+);
+
+CREATE TABLE kkp_mapping_service (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
+ service_name VARCHAR(255) NOT NULL,
+ component_name VARCHAR(255) NOT NULL,
+ CONSTRAINT PK_kkp_mapping_service PRIMARY KEY (kkp_id, service_name, component_name),
+ CONSTRAINT FK_kkp_service_principal FOREIGN KEY (kkp_id) REFERENCES kerberos_keytab_principal (kkp_id)
);
CREATE TABLE kerberos_descriptor
@@ -1050,6 +1066,7 @@ CREATE INDEX idx_alert_history_state on alert_history(alert_state);
CREATE INDEX idx_alert_group_name on alert_group(group_name);
CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('kkp_id_seq', 0);
-- In order for the first ID to be 1, must initialize the ambari_sequences table with a sequence_value of 0.
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('cluster_id_seq', 1);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('host_id_seq', 0);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
index 8a88aba..86c1d6c 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
@@ -933,18 +933,34 @@ CREATE TABLE kerberos_principal (
CREATE TABLE kerberos_keytab (
keytab_path VARCHAR(255) NOT NULL,
- CONSTRAINT PK_krb_keytab_path_host_id PRIMARY KEY CLUSTERED (keytab_path)
+ owner_name VARCHAR(255),
+ owner_access VARCHAR(255),
+ group_name VARCHAR(255),
+ group_access VARCHAR(255),
+ is_ambari_keytab SMALLINT NOT NULL DEFAULT 0,
+ write_ambari_jaas SMALLINT NOT NULL DEFAULT 0,
+ CONSTRAINT PK_kerberos_keytab PRIMARY KEY CLUSTERED (keytab_path)
);
-CREATE TABLE kerberos_principal_host (
- principal_name VARCHAR(255) NOT NULL,
+CREATE TABLE kerberos_keytab_principal (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
keytab_path VARCHAR(255) NOT NULL,
+ principal_name VARCHAR(255) NOT NULL,
+ host_id BIGINT,
is_distributed SMALLINT NOT NULL DEFAULT 0,
- host_id BIGINT NOT NULL,
- CONSTRAINT PK_kerberos_principal_host PRIMARY KEY CLUSTERED (principal_name, keytab_path, host_id),
- CONSTRAINT FK_krb_pr_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
- CONSTRAINT FK_krb_pr_host_principalname FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
- CONSTRAINT FK_krb_pr_host_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path)
+ CONSTRAINT PK_kkp PRIMARY KEY CLUSTERED (kkp_id),
+ CONSTRAINT FK_kkp_keytab_path FOREIGN KEY (keytab_path) REFERENCES kerberos_keytab (keytab_path),
+ CONSTRAINT FK_kkp_host_id FOREIGN KEY (host_id) REFERENCES hosts (host_id),
+ CONSTRAINT FK_kkp_principal_name FOREIGN KEY (principal_name) REFERENCES kerberos_principal (principal_name),
+ CONSTRAINT UNI_kkp UNIQUE(keytab_path, principal_name, host_id)
+);
+
+CREATE TABLE kkp_mapping_service (
+ kkp_id BIGINT NOT NULL DEFAULT 0,
+ service_name VARCHAR(255) NOT NULL,
+ component_name VARCHAR(255) NOT NULL,
+ CONSTRAINT PK_kkp_mapping_service PRIMARY KEY CLUSTERED (kkp_id, service_name, component_name),
+ CONSTRAINT FK_kkp_service_principal FOREIGN KEY (kkp_id) REFERENCES kerberos_keytab_principal (kkp_id)
);
CREATE TABLE kerberos_descriptor
@@ -1078,6 +1094,7 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
BEGIN TRANSACTION
INSERT INTO ambari_sequences (sequence_name, [sequence_value])
VALUES
+ ('kkp_id_seq', 0),
('cluster_id_seq', 1),
('host_id_seq', 0),
('user_id_seq', 2),
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/META-INF/persistence.xml b/ambari-server/src/main/resources/META-INF/persistence.xml
index 686c831..a04a5a0 100644
--- a/ambari-server/src/main/resources/META-INF/persistence.xml
+++ b/ambari-server/src/main/resources/META-INF/persistence.xml
@@ -46,7 +46,8 @@
<class>org.apache.ambari.server.orm.entities.HostVersionEntity</class>
<class>org.apache.ambari.server.orm.entities.KerberosPrincipalEntity</class>
<class>org.apache.ambari.server.orm.entities.KerberosKeytabEntity</class>
- <class>org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity</class>
+ <class>org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity</class>
+ <class>org.apache.ambari.server.orm.entities.KerberosKeytabServiceMappingEntity</class>
<class>org.apache.ambari.server.orm.entities.KeyValueEntity</class>
<class>org.apache.ambari.server.orm.entities.MemberEntity</class>
<class>org.apache.ambari.server.orm.entities.MetainfoEntity</class>
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index b4ff5c1..c443739 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -30,7 +30,9 @@ import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HDFS;
import static org.apache.ambari.server.agent.DummyHeartbeatConstants.HDFS_CLIENT;
import static org.apache.ambari.server.agent.DummyHeartbeatConstants.NAMENODE;
import static org.apache.ambari.server.agent.DummyHeartbeatConstants.SECONDARY_NAMENODE;
+import static org.apache.ambari.server.controller.KerberosHelperImpl.REMOVE_KEYTAB;
import static org.apache.ambari.server.controller.KerberosHelperImpl.SET_KEYTAB;
+import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
@@ -47,6 +49,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -79,8 +82,10 @@ import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
import org.apache.ambari.server.orm.OrmTestHelper;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileWriter;
-import org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileWriterFactory;
import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Alert;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
@@ -108,6 +113,7 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Sets;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
@@ -1428,8 +1434,6 @@ public class TestHeartbeatHandler {
properties = kcp.get(0);
Assert.assertNotNull(properties);
Assert.assertEquals("c6403.ambari.apache.org", properties.get(KerberosIdentityDataFileWriter.HOSTNAME));
- Assert.assertEquals("HDFS", properties.get(KerberosIdentityDataFileWriter.SERVICE));
- Assert.assertEquals("DATANODE", properties.get(KerberosIdentityDataFileWriter.COMPONENT));
Assert.assertEquals("dn/_HOST@_REALM", properties.get(KerberosIdentityDataFileWriter.PRINCIPAL));
Assert.assertEquals("/etc/security/keytabs/dn.service.keytab", properties.get(KerberosIdentityDataFileWriter.KEYTAB_FILE_PATH));
Assert.assertEquals("hdfs", properties.get(KerberosIdentityDataFileWriter.KEYTAB_FILE_OWNER_NAME));
@@ -1448,8 +1452,6 @@ public class TestHeartbeatHandler {
properties = kcp.get(0);
Assert.assertNotNull(properties);
Assert.assertEquals("c6403.ambari.apache.org", properties.get(KerberosIdentityDataFileWriter.HOSTNAME));
- Assert.assertEquals("HDFS", properties.get(KerberosIdentityDataFileWriter.SERVICE));
- Assert.assertEquals("DATANODE", properties.get(KerberosIdentityDataFileWriter.COMPONENT));
Assert.assertEquals("dn/_HOST@_REALM", properties.get(KerberosIdentityDataFileWriter.PRINCIPAL));
Assert.assertEquals("/etc/security/keytabs/dn.service.keytab", properties.get(KerberosIdentityDataFileWriter.KEYTAB_FILE_PATH));
Assert.assertFalse(properties.containsKey(KerberosIdentityDataFileWriter.KEYTAB_FILE_OWNER_NAME));
@@ -1481,7 +1483,6 @@ public class TestHeartbeatHandler {
Map<String, String> commandparams = new HashMap<>();
commandparams.put(KerberosServerAction.AUTHENTICATED_USER_NAME, "admin");
- commandparams.put(KerberosServerAction.DATA_DIRECTORY, createTestKeytabData().getAbsolutePath());
executionCommand.setCommandParams(commandparams);
ActionQueue aq = new ActionQueue();
@@ -1496,7 +1497,10 @@ public class TestHeartbeatHandler {
}});
replay(am);
- heartbeatTestHelper.getHeartBeatHandler(am, aq).injectKeytab(executionCommand, SET_KEYTAB, targetHost);
+
+ HeartBeatHandler handler = heartbeatTestHelper.getHeartBeatHandler(am, aq);
+ commandparams.put(KerberosServerAction.DATA_DIRECTORY, createTestKeytabData(handler).getAbsolutePath());
+ handler.injectKeytab(executionCommand, SET_KEYTAB, targetHost);
return executionCommand.getKerberosCommandParams();
}
@@ -1512,7 +1516,6 @@ public class TestHeartbeatHandler {
Map<String, String> commandparams = new HashMap<>();
commandparams.put(KerberosServerAction.AUTHENTICATED_USER_NAME, "admin");
- commandparams.put(KerberosServerAction.DATA_DIRECTORY, createTestKeytabData().getAbsolutePath());
executionCommand.setCommandParams(commandparams);
ActionQueue aq = new ActionQueue();
@@ -1527,38 +1530,60 @@ public class TestHeartbeatHandler {
}});
replay(am);
- heartbeatTestHelper.getHeartBeatHandler(am, aq).injectKeytab(executionCommand, "REMOVE_KEYTAB", targetHost);
+ HeartBeatHandler handler = heartbeatTestHelper.getHeartBeatHandler(am, aq);
+ commandparams.put(KerberosServerAction.DATA_DIRECTORY, createTestKeytabData(handler).getAbsolutePath());
+ handler.injectKeytab(executionCommand, REMOVE_KEYTAB, targetHost);
return executionCommand.getKerberosCommandParams();
}
- private File createTestKeytabData() throws Exception {
+ private File createTestKeytabData(HeartBeatHandler heartbeatHandler) throws Exception {
+ KerberosKeytabController kerberosKeytabControllerMock = createMock(KerberosKeytabController.class);
+ expect(kerberosKeytabControllerMock.getFilteredKeytabs(null,null,null)).andReturn(
+ Sets.newHashSet(
+ new ResolvedKerberosKeytab(
+ "/etc/security/keytabs/dn.service.keytab",
+ "hdfs",
+ "r",
+ "hadoop",
+ "",
+ Sets.newHashSet(new ResolvedKerberosPrincipal(
+ 1L,
+ "c6403.ambari.apache.org",
+ "dn/_HOST@_REALM",
+ false,
+ "/tmp",
+ "HDFS",
+ "DATANODE",
+ "/etc/security/keytabs/dn.service.keytab"
+ )
+ ),
+ false,
+ false
+ )
+ )
+ ).once();
+
+ replay(kerberosKeytabControllerMock);
+
+ Field controllerField = heartbeatHandler.getClass().getDeclaredField("kerberosKeytabController");
+ controllerField.setAccessible(true);
+ controllerField.set(heartbeatHandler, kerberosKeytabControllerMock);
+
File dataDirectory = temporaryFolder.newFolder();
- File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
- KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = injector.getInstance(KerberosIdentityDataFileWriterFactory.class).createKerberosIdentityDataFileWriter(identityDataFile);
File hostDirectory = new File(dataDirectory, "c6403.ambari.apache.org");
-
File keytabFile;
if(hostDirectory.mkdirs()) {
- keytabFile = new File(hostDirectory, DigestUtils.sha1Hex("/etc/security/keytabs/dn.service.keytab"));
+ keytabFile = new File(hostDirectory, DigestUtils.sha256Hex("/etc/security/keytabs/dn.service.keytab"));
+ FileWriter fw = new FileWriter(keytabFile);
+ BufferedWriter bw = new BufferedWriter(fw);
+ bw.write("hello");
+ bw.close();
} else {
throw new Exception("Failed to create " + hostDirectory.getAbsolutePath());
}
- kerberosIdentityDataFileWriter.writeRecord("c6403.ambari.apache.org", "HDFS", "DATANODE",
- "dn/_HOST@_REALM", "service",
- "/etc/security/keytabs/dn.service.keytab",
- "hdfs", "r", "hadoop", "", "false");
-
- kerberosIdentityDataFileWriter.close();
-
- // Ensure the host directory exists...
- FileWriter fw = new FileWriter(keytabFile);
- BufferedWriter bw = new BufferedWriter(fw);
- bw.write("hello");
- bw.close();
-
return dataDirectory;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
index ee87d24..1bee291 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -79,7 +79,9 @@ import org.apache.ambari.server.metadata.RoleCommandOrderProvider;
import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.orm.dao.ArtifactDAO;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.security.SecurityHelper;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.security.encryption.CredentialStoreService;
@@ -97,6 +99,7 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException
import org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler;
import org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandlerFactory;
import org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.stack.StackManagerFactory;
import org.apache.ambari.server.stageplanner.RoleGraphFactory;
import org.apache.ambari.server.state.Cluster;
@@ -258,6 +261,7 @@ public class KerberosHelperTest extends EasyMockSupport {
bind(AuditLogger.class).toInstance(createNiceMock(AuditLogger.class));
bind(ArtifactDAO.class).toInstance(createNiceMock(ArtifactDAO.class));
bind(KerberosPrincipalDAO.class).toInstance(createNiceMock(KerberosPrincipalDAO.class));
+ bind(KerberosKeytabPrincipalDAO.class).toInstance(createNiceMock(KerberosKeytabPrincipalDAO.class));
bind(RoleCommandOrderProvider.class).to(CachedRoleCommandOrderProvider.class);
bind(HostRoleCommandFactory.class).to(HostRoleCommandFactoryImpl.class);
@@ -2673,9 +2677,10 @@ public class KerberosHelperTest extends EasyMockSupport {
expect(kerberosDescriptor.getService("SERVICE1")).andReturn(service1KerberosDescriptor).times(1);
expect(kerberosDescriptor.getService("SERVICE2")).andReturn(service2KerberosDescriptor).times(1);
+ Capture<ResolvedKerberosPrincipal> spnegoPrincipalCapture = newCapture(CaptureType.LAST);
+ Capture<ResolvedKerberosPrincipal> ambariPrincipalCapture = newCapture(CaptureType.LAST);
+ String spnegoPrincipalNameExpected = String.format("HTTP/%s@%s", ambariServerHostname, realm);
if (createAmbariIdentities) {
- String spnegoPrincipalNameExpected = String.format("HTTP/%s@%s", ambariServerHostname, realm);
-
ArrayList<KerberosIdentityDescriptor> ambarServerComponent1Identities = new ArrayList<>();
ambarServerComponent1Identities.add(createMockIdentityDescriptor(
KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME,
@@ -2687,23 +2692,24 @@ public class KerberosHelperTest extends EasyMockSupport {
createMockPrincipalDescriptor("HTTP/_HOST@${realm}", KerberosPrincipalType.SERVICE, null, null),
createMockKeytabDescriptor("spnego.service.keytab", null)));
- KerberosComponentDescriptor ambariServerComponentKerberosDescriptor = createMockComponentDescriptor("AMBARI_SERVER", ambarServerComponent1Identities, null);
+ KerberosComponentDescriptor ambariServerComponentKerberosDescriptor = createMockComponentDescriptor(RootComponent.AMBARI_SERVER.name(), ambarServerComponent1Identities, null);
HashMap<String, KerberosComponentDescriptor> ambariServerComponentDescriptorMap = new HashMap<>();
- ambariServerComponentDescriptorMap.put("AMBARI_SERVER", ambariServerComponentKerberosDescriptor);
+ ambariServerComponentDescriptorMap.put(RootComponent.AMBARI_SERVER.name(), ambariServerComponentKerberosDescriptor);
- KerberosServiceDescriptor ambariServiceKerberosDescriptor = createMockServiceDescriptor("AMBARI", ambariServerComponentDescriptorMap, null, false);
- expect(ambariServiceKerberosDescriptor.getComponent("AMBARI_SERVER")).andReturn(ambariServerComponentKerberosDescriptor).once();
+ KerberosServiceDescriptor ambariServiceKerberosDescriptor = createMockServiceDescriptor(RootService.AMBARI.name(), ambariServerComponentDescriptorMap, null, false);
+ expect(ambariServiceKerberosDescriptor.getComponent(RootComponent.AMBARI_SERVER.name())).andReturn(ambariServerComponentKerberosDescriptor).once();
- expect(kerberosDescriptor.getService("AMBARI")).andReturn(ambariServiceKerberosDescriptor).once();
+ expect(kerberosDescriptor.getService(RootService.AMBARI.name())).andReturn(ambariServiceKerberosDescriptor).once();
ConfigureAmbariIdentitiesServerAction configureAmbariIdentitiesServerAction = injector.getInstance(ConfigureAmbariIdentitiesServerAction.class);
- expect(configureAmbariIdentitiesServerAction.installAmbariServerIdentity(eq(ambariServerPrincipalNameExpected), anyString(), eq(ambariServerKeytabFilePath),
- eq("user1"), eq(true), eq(true), eq("groupA"), eq(true), eq(false), (ActionLog) eq(null)))
+
+ expect(configureAmbariIdentitiesServerAction.installAmbariServerIdentity(capture(ambariPrincipalCapture), anyString(), eq(ambariServerKeytabFilePath),
+ eq("user1"), eq("rw"), eq("groupA"), eq("r"), (ActionLog) eq(null)))
.andReturn(true)
.once();
- expect(configureAmbariIdentitiesServerAction.installAmbariServerIdentity(eq(spnegoPrincipalNameExpected), anyString(), eq("spnego.service.keytab"),
- eq("user1"), eq(true), eq(true), eq("groupA"), eq(true), eq(false), (ActionLog) eq(null)))
+ expect(configureAmbariIdentitiesServerAction.installAmbariServerIdentity(capture(spnegoPrincipalCapture), anyString(), eq("spnego.service.keytab"),
+ eq("user1"), eq("rw"), eq("groupA"), eq("r"), (ActionLog) eq(null)))
.andReturn(true)
.once();
@@ -2765,6 +2771,11 @@ public class KerberosHelperTest extends EasyMockSupport {
verifyAll();
+ if (createAmbariIdentities) {
+ assertEquals(ambariPrincipalCapture.getValue().getPrincipal(), ambariServerPrincipalNameExpected);
+ assertEquals(spnegoPrincipalCapture.getValue().getPrincipal(), spnegoPrincipalNameExpected);
+ }
+
List<? extends String> capturedPrincipals = capturePrincipal.getValues();
assertEquals(createAmbariIdentities ? 5 : 3, capturedPrincipals.size());
assertTrue(capturedPrincipals.contains("s1_1@EXAMPLE.COM"));
@@ -3424,6 +3435,8 @@ public class KerberosHelperTest extends EasyMockSupport {
private void testCreateTestIdentity(final PrincipalKeyCredential PrincipalKeyCredential, Boolean manageIdentities) throws Exception {
KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
+ KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO = injector.getInstance(KerberosKeytabPrincipalDAO.class);
+ expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), anyObject(), anyObject())).andReturn(createNiceMock(KerberosKeytabPrincipalEntity.class)).anyTimes();
boolean managingIdentities = !Boolean.FALSE.equals(manageIdentities);
final Map<String, String> kerberosEnvProperties = new HashMap<>();
@@ -3625,6 +3638,8 @@ public class KerberosHelperTest extends EasyMockSupport {
private void testDeleteTestIdentity(final PrincipalKeyCredential PrincipalKeyCredential) throws Exception {
KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
+ KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO = injector.getInstance(KerberosKeytabPrincipalDAO.class);
+ expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), anyObject(), anyObject())).andReturn(createNiceMock(KerberosKeytabPrincipalEntity.class)).anyTimes();
Host host1 = createMock(Host.class);
expect(host1.getHostId()).andReturn(1l).anyTimes();
@@ -4093,13 +4108,13 @@ public class KerberosHelperTest extends EasyMockSupport {
createMockKeytabDescriptor("spnego.service.keytab", null)));
HashMap<String, KerberosComponentDescriptor> ambariServerComponentDescriptorMap = new HashMap<>();
- KerberosComponentDescriptor componentDescrptor = createMockComponentDescriptor("AMBARI_SERVER", ambarServerComponent1Identities, null);
- ambariServerComponentDescriptorMap.put("AMBARI_SERVER", componentDescrptor);
+ KerberosComponentDescriptor componentDescrptor = createMockComponentDescriptor(RootComponent.AMBARI_SERVER.name(), ambarServerComponent1Identities, null);
+ ambariServerComponentDescriptorMap.put(RootComponent.AMBARI_SERVER.name(), componentDescrptor);
- KerberosServiceDescriptor ambariServiceKerberosDescriptor = createMockServiceDescriptor("AMBARI", ambariServerComponentDescriptorMap, null, false);
- expect(ambariServiceKerberosDescriptor.getComponent("AMBARI_SERVER")).andReturn(componentDescrptor).once();
+ KerberosServiceDescriptor ambariServiceKerberosDescriptor = createMockServiceDescriptor(RootService.AMBARI.name(), ambariServerComponentDescriptorMap, null, false);
+ expect(ambariServiceKerberosDescriptor.getComponent(RootComponent.AMBARI_SERVER.name())).andReturn(componentDescrptor).once();
- expect(kerberosDescriptor.getService("AMBARI")).andReturn(ambariServiceKerberosDescriptor).once();
+ expect(kerberosDescriptor.getService(RootService.AMBARI.name())).andReturn(ambariServiceKerberosDescriptor).once();
}
replayAll();
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
index 59fbba0..e271932 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
@@ -39,9 +39,10 @@ import org.apache.ambari.server.controller.spi.ResourceProvider;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.orm.dao.HostDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor;
@@ -189,8 +190,12 @@ public class HostKerberosIdentityResourceProviderTest extends EasyMockSupport {
expect(kerberosPrincipalDAO.exists("principal2/Host100@EXAMPLE.COM")).andReturn(true).times(1);
expect(kerberosPrincipalDAO.exists("principal5@EXAMPLE.COM")).andReturn(false).times(1);
- KerberosPrincipalHostDAO kerberosPrincipalHostDAO = createStrictMock(KerberosPrincipalHostDAO.class);
- expect(kerberosPrincipalHostDAO.exists("principal1@EXAMPLE.COM", 100L, "/etc/security/keytabs/principal1.headless.keytab")).andReturn(true).times(1);
+ KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO = createStrictMock(KerberosKeytabPrincipalDAO.class);
+ KerberosKeytabPrincipalEntity distributedEntity = new KerberosKeytabPrincipalEntity();
+ distributedEntity.setDistributed(true);
+ expect(kerberosKeytabPrincipalDAO.findByNaturalKey(100L,"/etc/security/keytabs/principal1.headless.keytab", "principal1@EXAMPLE.COM"))
+ .andReturn(distributedEntity)
+ .times(1);
HostEntity host100 = createStrictMock(HostEntity.class);
expect(host100.getHostId()).andReturn(100L).times(1);
@@ -228,9 +233,9 @@ public class HostKerberosIdentityResourceProviderTest extends EasyMockSupport {
field.setAccessible(true);
field.set(provider, kerberosPrincipalDAO);
- field = HostKerberosIdentityResourceProvider.class.getDeclaredField("kerberosPrincipalHostDAO");
+ field = HostKerberosIdentityResourceProvider.class.getDeclaredField("kerberosKeytabPrincipalDAO");
field.setAccessible(true);
- field.set(provider, kerberosPrincipalHostDAO);
+ field.set(provider, kerberosKeytabPrincipalDAO);
field = HostKerberosIdentityResourceProvider.class.getDeclaredField("hostDAO");
field.setAccessible(true);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/orm/db/DDLTests.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/db/DDLTests.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/db/DDLTests.java
index 96cf64e..099400b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/orm/db/DDLTests.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/db/DDLTests.java
@@ -37,7 +37,7 @@ import com.google.common.collect.Sets;
/**
- * Test to check the sanity and conisistence of DDL scripts for different SQL dialects.
+ * Test to check the sanity and consistence of DDL scripts for different SQL dialects.
* (e.g. no unnamed constraints, the same tables with the same columns and constraints must exist)
*/
public class DDLTests {
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
index 39dee24..4cf3c72 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
@@ -19,7 +19,6 @@
package org.apache.ambari.server.serveraction.kerberos;
import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
@@ -31,11 +30,15 @@ import javax.persistence.EntityManager;
import org.apache.ambari.server.audit.AuditLogger;
import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.RootComponent;
+import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.orm.dao.HostDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.serveraction.ActionLog;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.io.FileUtils;
@@ -77,20 +80,22 @@ public class ConfigureAmbariIdentitiesServerActionTest extends EasyMockSupport {
Injector injector = createInjector();
+ HostDAO hostDAO = injector.getInstance(HostDAO.class);
+
HostEntity hostEntity;
if (ambariServerHasAgent) {
- KerberosPrincipalHostDAO kerberosPrincipalHostDAO = injector.getInstance(KerberosPrincipalHostDAO.class);
- expect(kerberosPrincipalHostDAO.exists(eq(principal), eq(1L), anyString())).andReturn(false).anyTimes();
- kerberosPrincipalHostDAO.create(anyObject());
- expectLastCall().anyTimes();
hostEntity = createMock(HostEntity.class);
expect(hostEntity.getHostId()).andReturn(1L).once();
+ expect(hostDAO.findById(1L)).andReturn(hostEntity).once();
} else {
hostEntity = null;
}
- HostDAO hostDAO = injector.getInstance(HostDAO.class);
expect(hostDAO.findByName(StageUtils.getHostName())).andReturn(hostEntity).once();
+ KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO = injector.getInstance(KerberosKeytabPrincipalDAO.class);
+ KerberosKeytabPrincipalEntity kke = createNiceMock(KerberosKeytabPrincipalEntity.class);
+ expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), eq(hostEntity), anyObject())).andReturn(kke).once();
+ expect(kerberosKeytabPrincipalDAO.merge(kke)).andReturn(createNiceMock(KerberosKeytabPrincipalEntity.class)).once();
// Mock the methods that do the actual file manipulation to avoid having to deal with ambari-sudo.sh used in
// ShellCommandUtil#mkdir, ShellCommandUtil#copyFile, etc..
@@ -113,8 +118,18 @@ public class ConfigureAmbariIdentitiesServerActionTest extends EasyMockSupport {
replayAll();
injector.injectMembers(action);
- action.installAmbariServerIdentity(principal, srcKeytabFile.getAbsolutePath(), destKeytabFile.getAbsolutePath(),
- "user1", true, true, "groupA", true, false, actionLog);
+ action.installAmbariServerIdentity(
+ new ResolvedKerberosPrincipal(
+ null,
+ null,
+ principal,
+ false,
+ null,
+ RootService.AMBARI.name(),
+ RootComponent.AMBARI_SERVER.name(),
+ destKeytabFile.getPath()
+ ), srcKeytabFile.getAbsolutePath(), destKeytabFile.getAbsolutePath(),
+ "user1", "rw", "groupA", "r", actionLog);
verifyAll();
@@ -200,7 +215,8 @@ public class ConfigureAmbariIdentitiesServerActionTest extends EasyMockSupport {
bind(KerberosHelper.class).toInstance(createNiceMock(KerberosHelper.class));
bind(HostDAO.class).toInstance(createMock(HostDAO.class));
- bind(KerberosPrincipalHostDAO.class).toInstance(createMock(KerberosPrincipalHostDAO.class));
+ bind(KerberosKeytabPrincipalDAO.class).toInstance(createMock(KerberosKeytabPrincipalDAO.class));
+// bind(KerberosPrincipalHostDAO.class).toInstance(createMock(KerberosPrincipalHostDAO.class));
}
});
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerActionTest.java
index c9301f3..724b634 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerActionTest.java
@@ -41,6 +41,7 @@ import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.audit.AuditLogger;
import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.RootComponent;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
@@ -48,6 +49,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.easymock.EasyMock;
import org.easymock.EasyMockSupport;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -64,6 +66,7 @@ public class FinalizeKerberosServerActionTest extends EasyMockSupport {
public TemporaryFolder folder = new TemporaryFolder();
@Test
+ @Ignore("Update accordingly to changes")
public void executeMITKDCOption() throws Exception {
String clusterName = "c1";
Injector injector = setup(clusterName);
@@ -154,7 +157,7 @@ public class FinalizeKerberosServerActionTest extends EasyMockSupport {
expect(executionCommand.getRoleCommand()).andReturn(RoleCommand.EXECUTE).anyTimes();
expect(executionCommand.getRole()).andReturn(Role.AMBARI_SERVER_ACTION.name()).anyTimes();
expect(executionCommand.getConfigurationTags()).andReturn(Collections.emptyMap()).anyTimes();
- expect(executionCommand.getServiceName()).andReturn("AMBARI_SERVER").anyTimes();
+ expect(executionCommand.getServiceName()).andReturn(RootComponent.AMBARI_SERVER.name()).anyTimes();
expect(executionCommand.getTaskId()).andReturn(3L).anyTimes();
return executionCommand;
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
index e6f0868..8e0ccae 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
@@ -40,6 +40,9 @@ import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.audit.AuditLogger;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.stack.OsFamily;
@@ -48,6 +51,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.collect.Sets;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -60,21 +64,36 @@ public class KerberosServerActionTest extends EasyMockSupport {
File temporaryDirectory;
private Injector injector;
private KerberosServerAction action;
+ private Cluster cluster;
+ private KerberosKeytabController kerberosKeytabController;
@Before
public void setUp() throws Exception {
- Cluster cluster = createMock(Cluster.class);
+ cluster = createMock(Cluster.class);
Clusters clusters = createMock(Clusters.class);
expect(clusters.getCluster(anyString())).andReturn(cluster).anyTimes();
ExecutionCommand mockExecutionCommand = createMock(ExecutionCommand.class);
HostRoleCommand mockHostRoleCommand = createMock(HostRoleCommand.class);
+ kerberosKeytabController = createMock(KerberosKeytabController.class);
+ expect(kerberosKeytabController.getFilteredKeytabs(null, null, null))
+ .andReturn(
+ Sets.newHashSet(new ResolvedKerberosKeytab(
+ null,
+ null,
+ null,
+ null,
+ null,
+ Sets.newHashSet(new ResolvedKerberosPrincipal(1l, "host", "principal", true, "/tmp", "SERVICE", "COMPONENT", "/tmp")),
+ true,
+ true))
+ ).anyTimes();
action = new KerberosServerAction() {
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
@@ -84,7 +103,7 @@ public class KerberosServerActionTest extends EasyMockSupport {
if (requestSharedDataContext.get("FAIL") != null) {
return createCommandReport(1, HostRoleStatus.FAILED, "{}", "ERROR", "ERROR");
} else {
- requestSharedDataContext.put(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL), evaluatedPrincipal);
+ requestSharedDataContext.put(resolvedPrincipal.getPrincipal(), resolvedPrincipal.getPrincipal());
return null;
}
}
@@ -110,6 +129,7 @@ public class KerberosServerActionTest extends EasyMockSupport {
bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
bind(AuditLogger.class).toInstance(createNiceMock(AuditLogger.class));
bind(KerberosOperationHandlerFactory.class).toInstance(createMock(KerberosOperationHandlerFactory.class));
+ bind(KerberosKeytabController.class).toInstance(kerberosKeytabController);
}
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
index a08f7a0..12a141b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
@@ -19,6 +19,7 @@
package org.apache.ambari.server.serveraction.upgrades;
import static org.apache.ambari.server.serveraction.upgrades.PreconfigureKerberosAction.UPGRADE_DIRECTION_KEY;
+import static org.easymock.EasyMock.anyLong;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.capture;
@@ -75,6 +76,7 @@ import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.UpgradeEntity;
import org.apache.ambari.server.security.encryption.CredentialStoreService;
@@ -111,6 +113,7 @@ import org.easymock.Capture;
import org.easymock.EasyMockSupport;
import org.easymock.IAnswer;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.gson.Gson;
@@ -170,13 +173,17 @@ public class PreconfigureKerberosActionTest extends EasyMockSupport {
verifyAll();
}
+ private Long hostId = 1L;
private Host createMockHost(String hostname) {
Host host = createNiceMock(Host.class);
expect(host.getHostName()).andReturn(hostname).anyTimes();
+ expect(host.getHostId()).andReturn(hostId).anyTimes();
+ hostId++;
return host;
}
@Test
+ @Ignore("Update accordingly to changes")
public void testUpgrade() throws Exception {
Capture<? extends Map<String, String>> captureCoreSiteProperties = newCapture();
@@ -184,9 +191,14 @@ public class PreconfigureKerberosActionTest extends EasyMockSupport {
HostDAO hostDAO = injector.getInstance(HostDAO.class);
EntityManager entityManager = injector.getInstance(EntityManager.class);
-
- expect(hostDAO.findByName(anyString())).andReturn(createNiceMock(HostEntity.class)).anyTimes();
+ HostEntity hostEntityMock = createNiceMock(HostEntity.class);
+ KerberosKeytabPrincipalEntity principalMock = createNiceMock(KerberosKeytabPrincipalEntity.class);
+ expect(principalMock.getHostId()).andReturn(1L).anyTimes();
+ expect(hostDAO.findByName(anyString())).andReturn(hostEntityMock).anyTimes();
+ expect(hostDAO.findById(anyLong())).andReturn(hostEntityMock).anyTimes();
expect(entityManager.find(eq(KerberosKeytabEntity.class), anyString())).andReturn(createNiceMock(KerberosKeytabEntity.class)).anyTimes();
+// expect(entityManager.find(eq(KerberosPrincipalHostEntity.class), anyObject())).andReturn(createNiceMock(KerberosPrincipalHostEntity.class)).anyTimes();
+ expect(entityManager.find(eq(KerberosKeytabPrincipalEntity.class), anyObject())).andReturn(principalMock).anyTimes();
ExecutionCommand executionCommand = createMockExecutionCommand(getDefaultCommandParams());
[42/51] [abbrv] ambari git commit: AMBARI-22115. Alerts for OneFS
mpack (amagyar)
Posted by am...@apache.org.
AMBARI-22115. Alerts for OneFS mpack (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/68e7dd3f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/68e7dd3f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/68e7dd3f
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 68e7dd3faad9523b0d7d177e041d94acb256e47b
Parents: d88c8ea
Author: Attila Magyar <am...@hortonworks.com>
Authored: Thu Oct 12 15:21:13 2017 +0200
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:58 2017 +0100
----------------------------------------------------------------------
ambari-server/docs/configuration/index.md | 1 +
.../server/alerts/JmxServerSideAlert.java | 104 +++++++++++++++++++
.../apache/ambari/server/alerts/Threshold.java | 73 +++++++++++++
.../server/configuration/Configuration.java | 11 ++
.../server/controller/ControllerModule.java | 1 +
.../server/controller/jmx/JMXMetricHolder.java | 27 +++++
.../org/apache/ambari/server/state/Alert.java | 1 +
.../ambari/server/state/alert/AlertUri.java | 41 ++++++++
.../ambari/server/state/alert/MetricSource.java | 7 ++
.../ambari/server/state/alert/Reporting.java | 43 ++++++++
.../ambari/server/state/alert/ServerSource.java | 15 +++
.../services/AmbariServerAlertService.java | 7 +-
.../ambari/server/alerts/ThresholdTest.java | 90 ++++++++++++++++
.../controller/jmx/JMXMetricHolderTest.java | 61 +++++++++++
.../ambari/server/state/alert/AlertUriTest.java | 60 +++++++++++
15 files changed, 541 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/docs/configuration/index.md
----------------------------------------------------------------------
diff --git a/ambari-server/docs/configuration/index.md b/ambari-server/docs/configuration/index.md
index 9dbe9c4..8f1531b 100644
--- a/ambari-server/docs/configuration/index.md
+++ b/ambari-server/docs/configuration/index.md
@@ -57,6 +57,7 @@ The following are the properties which can be used to configure Ambari.
| alerts.execution.scheduler.threadpool.size.core | The core number of threads used to process incoming alert events. The value should be increased as the size of the cluster increases. |`2` |
| alerts.execution.scheduler.threadpool.size.max | The number of threads used to handle alerts received from the Ambari Agents. The value should be increased as the size of the cluster increases. |`2` |
| alerts.execution.scheduler.threadpool.worker.size | The number of queued alerts allowed before discarding old alerts which have not been handled. The value should be increased as the size of the cluster increases. |`2000` |
+| alerts.server.side.scheduler.threadpool.size.core | The core pool size of the executor service that runs server side alerts. |`4` |
| alerts.snmp.dispatcher.udp.port | The UDP port to use when binding the SNMP dispatcher on Ambari Server startup. If no port is specified, then a random port will be used. | |
| alerts.template.file | The full path to the XML file that describes the different alert templates. | |
| ambari.display.url | The URL to use when creating messages which should include the Ambari Server URL.<br/><br/>The following are examples of valid values:<ul><li>`http://ambari.apache.org:8080`</ul> | |
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java b/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
new file mode 100644
index 0000000..a4b86f8
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/alerts/JmxServerSideAlert.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
+import org.apache.ambari.server.controller.jmx.JMXMetricHolder;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.alert.AlertDefinition;
+import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
+import org.apache.ambari.server.state.alert.MetricSource;
+import org.apache.ambari.server.state.alert.Reporting;
+import org.apache.ambari.server.state.alert.ServerSource;
+import org.apache.ambari.server.state.services.MetricsRetrievalService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+
+/**
+ * I represent a "SERVER" {@link org.apache.ambari.server.state.alert.SourceType} alert
+ * which can pull JMX metrics from a remote cluster.
+ */
+public class JmxServerSideAlert extends AlertRunnable {
+ private final static Logger LOG = LoggerFactory.getLogger(JmxServerSideAlert.class);
+ @Inject
+ private AlertDefinitionFactory definitionFactory;
+ @Inject
+ private MetricsRetrievalService metricsRetrievalService;
+ @Inject
+ private ConfigHelper configHelper;
+
+ public JmxServerSideAlert(String definitionName) {
+ super(definitionName);
+ }
+
+ @Override
+ List<Alert> execute(Cluster cluster, AlertDefinitionEntity entity) throws AmbariException {
+ AlertDefinition alertDef = definitionFactory.coerce(entity);
+ ServerSource serverSource = (ServerSource) alertDef.getSource();
+ URI jmxUrl = jmxUrl(cluster, serverSource);
+ JMXMetricHolder metricHolder = jmxMetric(serverSource, jmxUrl);
+ return metricHolder == null
+ ? emptyList()
+ : alerts(alertDef, serverSource.getJmxInfo(), metricHolder, serverSource.getReporting());
+ }
+
+ private URI jmxUrl(Cluster cluster, ServerSource serverSource) throws AmbariException {
+ return serverSource.getUri().resolve(config(cluster)).resolve(serverSource.getJmxInfo().getUrlSuffix());
+ }
+
+ private Map<String, Map<String, String>> config(Cluster cluster) throws AmbariException {
+ return configHelper.getEffectiveConfigProperties(cluster, configHelper.getEffectiveDesiredTags(cluster, null));
+ }
+
+ private JMXMetricHolder jmxMetric(ServerSource serverSource, URI jmxUri) {
+ URLStreamProvider streamProvider = new URLStreamProvider(
+ serverSource.getUri().getConnectionTimeoutMsec(),
+ serverSource.getUri().getReadTimeoutMsec(),
+ ComponentSSLConfiguration.instance());
+ metricsRetrievalService.submitRequest(MetricsRetrievalService.MetricSourceType.JMX, streamProvider, jmxUri.toString());
+ return metricsRetrievalService.getCachedJMXMetric(jmxUri.toString());
+ }
+
+ private List<Alert> alerts(AlertDefinition alertDef, MetricSource.JmxInfo jmxInfo, JMXMetricHolder jmxMetricHolder, Reporting reporting) throws AmbariException {
+ List<Object> metrics = jmxMetricHolder.findAll(jmxInfo.getPropertyList());
+ if (metrics.isEmpty()) {
+ return emptyList();
+ }
+ if (metrics.get(0) instanceof Number) {
+ Alert alert = reporting.alert(((Number) metrics.get(0)).doubleValue(), metrics, alertDef);
+ return singletonList(alert);
+ } else {
+ LOG.info("Unsupported metrics value: {} when running alert: {}", metrics.get(0), alertDef);
+ return emptyList();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/alerts/Threshold.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/alerts/Threshold.java b/ambari-server/src/main/java/org/apache/ambari/server/alerts/Threshold.java
new file mode 100644
index 0000000..ae33eeb
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/alerts/Threshold.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import javax.annotation.Nullable;
+
+import org.apache.ambari.server.state.AlertState;
+
+/**
+ * I'm a 3 level threshold where each level corresponds to ok, warn and critical levels.
+ * The levels are either increasing or decreasing numerical sequences.
+ * My main responsibility is to map incoming values to {@link AlertState} based on the threshold levels.
+ */
+public class Threshold {
+ private final Double okValue;
+ private final double warnValue;
+ private final double critValue;
+
+ public Threshold(@Nullable Double okValue, double warnValue, double critValue) {
+ this.okValue = okValue;
+ this.warnValue = warnValue;
+ this.critValue = critValue;
+ }
+
+ public AlertState state(double value) {
+ return directionUp() ? stateWhenDirectionUp(value) : stateWhenDirectionDown(value);
+ }
+
+ private boolean directionUp() {
+ return critValue >= warnValue;
+ }
+
+ private AlertState stateWhenDirectionUp(double value) {
+ if (value >= critValue) {
+ return AlertState.CRITICAL;
+ }
+ if (value >= warnValue) {
+ return AlertState.WARNING;
+ }
+ if (okValue == null || value >= okValue) {
+ return AlertState.OK;
+ }
+ return AlertState.UNKNOWN;
+ }
+
+ private AlertState stateWhenDirectionDown(double value) {
+ if (value <= critValue) {
+ return AlertState.CRITICAL;
+ }
+ if (value <= warnValue) {
+ return AlertState.WARNING;
+ }
+ if (okValue == null || value <= okValue) {
+ return AlertState.OK;
+ }
+ return AlertState.UNKNOWN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 7003c98..09d5799 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2786,6 +2786,13 @@ public class Configuration {
@Markdown(description = "Whether security password encryption is enabled or not. In case it is we store passwords in their own file(s); otherwise we store passwords in the Ambari credential store.")
public static final ConfigurationProperty<Boolean> SECURITY_PASSWORD_ENCRYPTON_ENABLED = new ConfigurationProperty<Boolean>("security.passwords.encryption.enabled", false);
+ /**
+ * The core pool size of the executor service that runs server side alerts.
+ */
+ @Markdown(description = "The core pool size of the executor service that runs server side alerts.")
+ public static final ConfigurationProperty<Integer> SERVER_SIDE_ALERTS_CORE_POOL_SIZE = new ConfigurationProperty<>(
+ "alerts.server.side.scheduler.threadpool.size.core", 4);
+
private static final Logger LOG = LoggerFactory.getLogger(
Configuration.class);
@@ -6234,4 +6241,8 @@ public class Configuration {
public String getAutoGroupCreation() {
return getProperty(AUTO_GROUP_CREATION);
}
+
+ public int getAlertServiceCorePoolSize() {
+ return Integer.parseInt(getProperty(SERVER_SIDE_ALERTS_CORE_POOL_SIZE));
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
index ed7513f..2938b26 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
@@ -359,6 +359,7 @@ public class ControllerModule extends AbstractModule {
// So it's an "additional time", given to stage to finish execution before
// it is considered as timed out
bindConstant().annotatedWith(Names.named("actionTimeout")).to(600000L);
+ bindConstant().annotatedWith(Names.named("alertServiceCorePoolSize")).to(configuration.getAlertServiceCorePoolSize());
bindConstant().annotatedWith(Names.named("dbInitNeeded")).to(dbInitNeeded);
bindConstant().annotatedWith(Names.named("statusCheckInterval")).to(5000L);
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
index 81d72fb..f6ae54f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXMetricHolder.java
@@ -18,13 +18,18 @@
package org.apache.ambari.server.controller.jmx;
+import static java.util.stream.Collectors.toList;
+
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
/**
*
*/
public final class JMXMetricHolder {
+ private static final String NAME_KEY = "name";
private List<Map<String, Object>> beans;
@@ -47,4 +52,26 @@ public final class JMXMetricHolder {
}
return stringBuilder.toString();
}
+
+ public List<Object> findAll(List<String> properties) {
+ return properties.stream()
+ .map(this::find)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(toList());
+ }
+
+ public Optional<Object> find(String property) {
+ String propertyName = property.split("/")[0];
+ String propertyValue = property.split("/")[1];
+ return beans.stream()
+ .filter(each -> propertyName.equals(name(each)))
+ .map(each -> each.get(propertyValue))
+ .filter(Objects::nonNull)
+ .findFirst();
+ }
+
+ private String name(Map<String, Object> bean) {
+ return bean.containsKey(NAME_KEY) ? (String) bean.get(NAME_KEY) : null;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/state/Alert.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Alert.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Alert.java
index 5d2ecc6..49bd5d2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Alert.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Alert.java
@@ -56,6 +56,7 @@ public class Alert {
component = componentName;
this.hostName = hostName;
state = alertState;
+ timestamp = System.currentTimeMillis();
}
public Alert() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java
index 93801d5..b3ef2de 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java
@@ -17,8 +17,13 @@
*/
package org.apache.ambari.server.state.alert;
+import java.net.URI;
+import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
+
import com.google.gson.annotations.SerializedName;
/**
@@ -87,6 +92,13 @@ public class AlertUri {
@SerializedName("connection_timeout")
private float m_connectionTimeout = 5.0f;
+
+ /**
+ * An optional read timeout value for connections.
+ */
+ @SerializedName("read_timeout")
+ private float readTimeout = 15.0f;
+
/**
* If present, then the component supports HA mode and the properties
* contained within need to be checked to see if an HA URI is required to be
@@ -117,6 +129,18 @@ public class AlertUri {
m_httpUri = httpUri;
}
+ public void setHttpsUri(String httpsUri) {
+ this.m_httpsUri = httpsUri;
+ }
+
+ public void setHttpsPropertyValue(String m_httpsPropertyValue) {
+ this.m_httpsPropertyValue = m_httpsPropertyValue;
+ }
+
+ public void setHttpsProperty(String m_httpsProperty) {
+ this.m_httpsProperty = m_httpsProperty;
+ }
+
/**
* Gets the default port to use on the host running the alert if none of the
* http properties are available.
@@ -276,6 +300,23 @@ public class AlertUri {
}
}
+ public URI resolve(Map<String, Map<String, String>> config) throws AmbariException {
+ VariableReplacementHelper variableReplacer = new VariableReplacementHelper();
+ String httpsProperty = variableReplacer.replaceVariables(m_httpsProperty, config);
+ String httpsPropertyValue = variableReplacer.replaceVariables(m_httpsPropertyValue, config);
+ return httpsProperty == null || !httpsProperty.equals(httpsPropertyValue)
+ ? URI.create(String.format("http://%s", variableReplacer.replaceVariables(m_httpUri, config)))
+ : URI.create(String.format("https://%s", variableReplacer.replaceVariables(m_httpsUri, config)));
+ }
+
+ public int getConnectionTimeoutMsec() {
+ return (int) m_connectionTimeout * 1000;
+ }
+
+ public int getReadTimeoutMsec() {
+ return (int) readTimeout * 1000;
+ }
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
index 11eee05..d7283fe 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/MetricSource.java
@@ -129,6 +129,9 @@ public class MetricSource extends Source {
private String value;
+ @SerializedName("url_suffix")
+ private String urlSuffix = "/jmx";
+
public List<String> getPropertyList() {
return propertyList;
}
@@ -152,5 +155,9 @@ public class MetricSource extends Source {
return list1.equals(list2);
}
+
+ public String getUrlSuffix() {
+ return urlSuffix;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
index 4aeba45..51d074e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
@@ -17,6 +17,15 @@
*/
package org.apache.ambari.server.state.alert;
+import static org.apache.ambari.server.state.alert.Reporting.ReportingType.PERCENT;
+
+import java.text.MessageFormat;
+import java.util.List;
+
+import org.apache.ambari.server.alerts.Threshold;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+
import com.google.gson.annotations.SerializedName;
/**
@@ -200,6 +209,40 @@ public class Reporting {
}
/**
+ * Map the incoming value to {@link AlertState} and generate an alert with that state.
+ */
+ public Alert alert(double value, List<Object> args, AlertDefinition alertDef) {
+ Alert alert = new Alert(alertDef.getName(), null, alertDef.getServiceName(), alertDef.getComponentName(), null, state(value));
+ alert.setText(MessageFormat.format(message(value), args.toArray()));
+ return alert;
+ }
+
+ private AlertState state(double value) {
+ return getThreshold().state(PERCENT == getType() ? value * 100 : value);
+ }
+
+ private Threshold getThreshold() {
+ return new Threshold(getOk().getValue(), getWarning().getValue(), getCritical().getValue());
+ }
+
+ private String message(double value) {
+ switch (state(value)) {
+ case OK:
+ return getOk().getText();
+ case WARNING:
+ return getWarning().getText();
+ case CRITICAL:
+ return getCritical().getText();
+ case UNKNOWN:
+ return "Unknown";
+ case SKIPPED:
+ return "Skipped";
+ default:
+ throw new IllegalStateException("Invalid alert state: " + state(value));
+ }
+ }
+
+ /**
* The {@link ReportTemplate} class is used to pair a label and threshhold
* value.
* <p/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java
index c58867a..765bdea 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java
@@ -28,6 +28,13 @@ public class ServerSource extends ParameterizedSource {
@SerializedName("class")
private String m_class;
+ @SerializedName("uri")
+ private AlertUri uri = null;
+
+ @SerializedName("jmx")
+ private MetricSource.JmxInfo jmxInfo = null;
+
+
/**
* Gets the fully qualified classname specified in the source.
*/
@@ -35,6 +42,14 @@ public class ServerSource extends ParameterizedSource {
return m_class;
}
+ public MetricSource.JmxInfo getJmxInfo() {
+ return jmxInfo;
+ }
+
+ public AlertUri getUri() {
+ return uri;
+ }
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
index 305f693..6cce1b2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
@@ -45,6 +45,7 @@ import com.google.common.util.concurrent.AbstractScheduledService;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
+import com.google.inject.name.Named;
/**
* The {@link AmbariServerAlertService} is used to manage the dynamically loaded
@@ -87,7 +88,7 @@ public class AmbariServerAlertService extends AbstractScheduledService {
/**
* The executor to use to run all {@link Runnable} alert classes.
*/
- private final ScheduledExecutorService m_scheduledExecutorService = Executors.newScheduledThreadPool(3);
+ private ScheduledExecutorService m_scheduledExecutorService;
/**
* A map of all of the definition names to {@link ScheduledFuture}s.
@@ -101,6 +102,10 @@ public class AmbariServerAlertService extends AbstractScheduledService {
public AmbariServerAlertService() {
}
+ @Inject
+ public void initExecutor(@Named("alertServiceCorePoolSize") int alertServiceCorePoolSize) {
+ this.m_scheduledExecutorService = Executors.newScheduledThreadPool(alertServiceCorePoolSize);
+ }
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/test/java/org/apache/ambari/server/alerts/ThresholdTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/alerts/ThresholdTest.java b/ambari-server/src/test/java/org/apache/ambari/server/alerts/ThresholdTest.java
new file mode 100644
index 0000000..07bfa03
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/alerts/ThresholdTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import static org.apache.ambari.server.state.AlertState.CRITICAL;
+import static org.apache.ambari.server.state.AlertState.OK;
+import static org.apache.ambari.server.state.AlertState.UNKNOWN;
+import static org.apache.ambari.server.state.AlertState.WARNING;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import org.apache.ambari.server.state.AlertState;
+import org.junit.Test;
+
+public class ThresholdTest {
+ @Test
+ public void testBetweenOkAndWarnIsOk_dirUp() throws Exception {
+ Threshold threshold = new Threshold(10.0, 20.0, 30.0);
+ assertState(threshold, OK, 10, 15, 19);
+ }
+
+ @Test
+ public void testBetweenWarnAndCritIsWarn_dirUp() throws Exception {
+ Threshold threshold = new Threshold(10.0, 20.0, 30.0);
+ assertState(threshold, WARNING, 20, 25, 29);
+ }
+
+ @Test
+ public void testAboveCritIsCrit_dirUp() throws Exception {
+ Threshold threshold = new Threshold(10.0, 20.0, 30.0);
+ assertState(threshold, CRITICAL, 30, 40, 99999);
+ }
+
+ @Test
+ public void testBelowOkIsUnknown_dirUp() throws Exception {
+ Threshold threshold = new Threshold(10.0, 20, 30);
+ assertState(threshold, UNKNOWN, 9, 2, -99999);
+ }
+
+ @Test
+ public void testBelowCritIsCrit_dirDown() throws Exception {
+ Threshold threshold = new Threshold(40.0, 30.0, 20.0);
+ assertState(threshold, CRITICAL, 20, 15, 2, -99999);
+ }
+
+ @Test
+ public void testBetweenWarnAndCritIsWarn_dirDown() throws Exception {
+ Threshold threshold = new Threshold(40.0, 30.0, 20.0);
+ assertState(threshold, WARNING, 30, 25, 21);
+ }
+
+ @Test
+ public void testBetweenOkAndWarnIsOk_dirDown() throws Exception {
+ Threshold threshold = new Threshold(40.0, 30.0, 20.0);
+ assertState(threshold, OK, 40, 35, 31);
+ }
+
+ @Test
+ public void testAboveOkIsUnknown_dirDown() throws Exception {
+ Threshold threshold = new Threshold(40.0, 30.0, 20.0);
+ assertState(threshold, UNKNOWN, 41, 50, 9999);
+ }
+
+ @Test
+ public void testOkIsOptional() throws Exception {
+ Threshold threshold = new Threshold(null, 20.0, 30.0);
+ assertState(threshold, OK, 10, 15, 19);
+ }
+
+ private void assertState(Threshold threshold, AlertState expectedState, int... values) {
+ for (int value: values) {
+ assertThat(expectedState, is(threshold.state(value)));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
new file mode 100644
index 0000000..61d5bae
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXMetricHolderTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.jmx;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.internal.matchers.IsCollectionContaining.hasItems;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class JMXMetricHolderTest {
+ private JMXMetricHolder metrics = new JMXMetricHolder();
+
+ @Before
+ public void setUp() {
+ metrics.setBeans(asList(
+ new HashMap<String, Object>() {{
+ put("name", "bean1");
+ put("value", "val1");
+ }},
+ new HashMap<String, Object>() {{
+ put("name", "bean2");
+ put("value", "val2");
+ }}
+ ));
+ }
+
+ @Test
+ public void testFindSingleBeanByName() throws Exception {
+ assertThat(metrics.find("bean1/value"), is(Optional.of("val1")));
+ assertThat(metrics.find("bean2/value"), is(Optional.of("val2")));
+ assertThat(metrics.find("bean3/notfound"), is(Optional.empty()));
+ }
+
+ @Test
+ public void testFindMultipleBeansByName() throws Exception {
+ List<Object> result = metrics.findAll(asList("bean1/value", "bean2/value", "bean3/notfound"));
+ assertThat(result, hasItems("val1", "val2"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/68e7dd3f/ambari-server/src/test/java/org/apache/ambari/server/state/alert/AlertUriTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/alert/AlertUriTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/alert/AlertUriTest.java
new file mode 100644
index 0000000..295e364
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/alert/AlertUriTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.alert;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.server.AmbariException;
+import org.junit.Test;
+
+public class AlertUriTest {
+ @Test
+ public void testChoosesHttpByDefault() throws Exception {
+ AlertUri uri = new AlertUri();
+ uri.setHttpUri("${config1/http-host}/path");
+ assertThat(resolved(uri), is("http://http-host/path"));
+ }
+
+ @Test
+ public void testChoosesHttpsBasedOnProperties() throws Exception {
+ AlertUri uri = new AlertUri();
+ uri.setHttpUri("${config1/http-host}/path");
+ uri.setHttpsUri("${config1/https-host}/path");
+ uri.setHttpsProperty("${config1/use-http}");
+ uri.setHttpsPropertyValue("YES");
+ assertThat(resolved(uri), is("https://https-host/path"));
+ }
+
+ private Map<String, Map<String, String>> config() {
+ return new HashMap<String, Map<String, String>>() {{
+ put("config1", new HashMap<String, String>() {{
+ put("http-host", "http-host");
+ put("https-host", "https-host");
+ put("use-http", "YES");
+ }});
+ }};
+ }
+
+ private String resolved(AlertUri uri) throws AmbariException {
+ return uri.resolve(config()).toString();
+ }
+}
\ No newline at end of file
[23/51] [abbrv] ambari git commit: AMBARI-22675 LogSearch Title Bar
Fixes. (Istvan Tobias via ababiichuk)
Posted by am...@apache.org.
AMBARI-22675 LogSearch Title Bar Fixes. (Istvan Tobias via ababiichuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f3dc1ca3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f3dc1ca3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f3dc1ca3
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: f3dc1ca399c25c3f875bf52e75948839ce5d0b11
Parents: 9bb0980
Author: Istvan Tobias <to...@gmail.com>
Authored: Tue Dec 19 19:14:17 2017 +0200
Committer: ababiichuk <ab...@hortonworks.com>
Committed: Tue Dec 19 19:14:17 2017 +0200
----------------------------------------------------------------------
.../src/app/components/app.component.html | 11 ++----
.../src/app/components/app.component.less | 4 +--
.../filters-panel/filters-panel.component.ts | 8 +++--
.../logs-container.component.html | 6 ++--
.../logs-container.component.less | 17 ++++++++--
.../logs-container/logs-container.component.ts | 35 +++++++++++++++++++-
.../main-container.component.html | 1 -
.../main-container.component.less | 1 -
.../main-container/main-container.component.ts | 5 +--
9 files changed, 63 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.html
index 833f43f..495c8d9 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.html
@@ -16,7 +16,7 @@
-->
<header>
- <nav class="navbar navbar-fixed-top">
+ <nav class="navbar">
<div class="container-fluid">
<h1 [ngClass]="{'full-flex-width': !isAuthorized, 'pull-left': true}">{{'common.title' | translate}}</h1>
<top-menu *ngIf="isAuthorized"></top-menu>
@@ -24,11 +24,4 @@
</nav>
</header>
-<main-container>
- <ng-template>
- <!-- hidden element for moving the main bar outside the fixed header -->
- <div class="navbar invisible">
- <h1> </h1>
- </div>
- </ng-template>
-</main-container>
+<main-container></main-container>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.less
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.less
index f0fecfc..476465c 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/app.component.less
@@ -18,11 +18,11 @@
@import 'mixins';
:host {
- .full-size;
+ background-color: @main-background-color; // TODO implement actual color
display: flex;
flex-direction: column;
- background-color: @main-background-color; // TODO implement actual color
line-height: @default-line-height;
+ min-height: 100vh;
.navbar {
margin-bottom: 0;
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
index f9fe94b..480706a 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import {Component, OnChanges, SimpleChanges, Input} from '@angular/core';
+import {Component, OnChanges, SimpleChanges, Input, ViewContainerRef} from '@angular/core';
import {FormGroup} from '@angular/forms';
import {Observable} from 'rxjs/Observable';
import {Subject} from 'rxjs/Subject';
@@ -33,7 +33,7 @@ import {LogsContainerService} from '@app/services/logs-container.service';
})
export class FiltersPanelComponent implements OnChanges {
- constructor(private logsContainer: LogsContainerService) {
+ constructor(private logsContainer: LogsContainerService, public viewContainerRef: ViewContainerRef) {
}
ngOnChanges(changes: SimpleChanges): void {
@@ -62,6 +62,10 @@ export class FiltersPanelComponent implements OnChanges {
searchBoxItems: Observable<ListItem[]>;
+ get containerEl(): Element {
+ return this.viewContainerRef.element.nativeElement;
+ }
+
get filters(): {[key: string]: FilterCondition} {
return this.logsContainer.filters;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.html
index 5e401d8..d1b11e6 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.html
@@ -24,9 +24,9 @@
</div>
</div>
</div>
-<div class="container-fluid">
- <filters-panel class="row" [filtersForm]="filtersForm" [logsType]="logsType"></filters-panel>
- <div class="row">
+<div #container [ngClass]="{'container-fluid': true, 'fixed-filterbar': isFilterPanelFixedPostioned}">
+ <filters-panel class="row" [filtersForm]="filtersForm" [logsType]="logsType" #filtersPanel></filters-panel>
+ <div class="row events-count">
<div *ngIf="autoRefreshRemainingSeconds" class="col-md-12">
<div class="auto-refresh-message pull-right">
{{'filter.capture.triggeringRefresh' | translate: autoRefreshMessageParams}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.less
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.less
index b4d44fb..243bb5b 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.less
@@ -32,7 +32,20 @@
}
}
- filters-panel {
- margin-bottom: @block-margin-top;
+ .fixed-filterbar {
+ filters-panel {
+ background-color: fadeout(@filters-panel-background-color, 35%);
+ left: 0;
+ margin: 0;
+ position: fixed;
+ top: 0;
+ width: 100%;
+ z-index: 1;
+ }
+ }
+
+ .events-count {
+ margin-top: @block-margin-top;
}
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
index 86709fb..cf28a8b 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import {Component} from '@angular/core';
+import {Component, ElementRef, ViewChild, HostListener} from '@angular/core';
import {FormGroup} from '@angular/forms';
import {Observable} from 'rxjs/Observable';
import {LogsContainerService} from '@app/services/logs-container.service';
@@ -31,6 +31,7 @@ import {ActiveServiceLogEntry} from '@app/classes/active-service-log-entry';
import {HistogramOptions} from '@app/classes/histogram-options';
import {ListItem} from '@app/classes/list-item';
import {LogsType} from '@app/classes/string';
+import {FiltersPanelComponent} from "@app/components/filters-panel/filters-panel.component";
@Component({
selector: 'logs-container',
@@ -51,6 +52,16 @@ export class LogsContainerComponent {
appState.getParameter('isServiceLogContextView').subscribe((value: boolean) => this.isServiceLogContextView = value);
}
+ @ViewChild('container') containerRef: ElementRef;
+ @ViewChild('filtersPanel') filtersPanelRef: FiltersPanelComponent;
+
+ @HostListener("window:scroll", ['$event'])
+ onWindowScroll(): void {
+ this.setFixedPositionValue();
+ }
+
+ private isFilterPanelFixedPostioned: boolean = false;
+
tabs: Observable<Tab[]> = this.tabsStorage.getAll();
get filtersForm(): FormGroup {
@@ -115,6 +126,28 @@ export class LogsContainerComponent {
return this.logsContainer.serviceLogsColumns;
}
+ /**
+ * The goal is to set the fixed position of the filter panel when it is scrolled to the top. So that the panel
+ * can be always visible for the user.
+ */
+ private setFixedPositionValue(): void {
+ const el:Element = this.containerRef.nativeElement;
+ const top:number = el.getBoundingClientRect().top;
+ const valueBefore: boolean = this.isFilterPanelFixedPostioned;
+ if (valueBefore != (top <= 0)) {
+ const fpEl:Element = this.filtersPanelRef.containerEl;
+ this.isFilterPanelFixedPostioned = top <= 0;
+ const filtersPanelHeight: number = fpEl.getBoundingClientRect().height;
+ const containerPaddingTop: number = parseFloat(window.getComputedStyle(el).paddingTop);
+ const htmlEl:HTMLElement = this.containerRef.nativeElement;
+ if (this.isFilterPanelFixedPostioned) {
+ htmlEl.style.paddingTop = (containerPaddingTop + filtersPanelHeight) + 'px';
+ } else {
+ htmlEl.style.paddingTop = (containerPaddingTop - filtersPanelHeight) + 'px';
+ }
+ }
+ }
+
setCustomTimeRange(startTime: number, endTime: number): void {
this.logsContainer.setCustomTimeRange(startTime, endTime);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.html
index 95dd238..b85cd87 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.html
@@ -15,7 +15,6 @@
limitations under the License.
-->
-<ng-template [ngTemplateOutlet]="template"></ng-template>
<div *ngIf="isInitialLoading" class="text-center">
<span class="fa fa-spinner fa-spin"></span>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.less
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.less
index bca668d..115f822 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.less
@@ -19,6 +19,5 @@
@import '../mixins';
:host {
- .full-size;
overflow-x: hidden;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3dc1ca3/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.ts
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.ts
index 6747a0c..f83d22b 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/main-container/main-container.component.ts
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import {Component, ContentChild, TemplateRef} from '@angular/core';
+import {Component} from '@angular/core';
import {AppStateService} from '@app/services/storage/app-state.service';
@Component({
@@ -31,9 +31,6 @@ export class MainContainerComponent {
appState.getParameter('isInitialLoading').subscribe((value: boolean) => this.isInitialLoading = value);
}
- @ContentChild(TemplateRef)
- template;
-
isAuthorized: boolean = false;
isInitialLoading: boolean = false;
[05/51] [abbrv] ambari git commit: AMBARI-22648: zeppelin server
keytab missing from zeppelin-site.xml (Bikas Saha via jluniya)
Posted by am...@apache.org.
AMBARI-22648: zeppelin server keytab missing from zeppelin-site.xml (Bikas Saha via jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d95d3ebe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d95d3ebe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d95d3ebe
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: d95d3ebeaf9e5334828ebc5d16fa223895f49f36
Parents: dc31e51
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Thu Dec 14 16:41:43 2017 -0800
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Thu Dec 14 16:41:43 2017 -0800
----------------------------------------------------------------------
.../common-services/ZEPPELIN/0.7.0/package/scripts/master.py | 8 +++++++-
.../src/test/python/stacks/2.6/configs/default.json | 2 ++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d95d3ebe/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
index ee264a4..df892f8 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
@@ -137,9 +137,15 @@ class Master(Script):
self.chown_zeppelin_pid_dir(env)
# write out zeppelin-site.xml
+ my_map = {}
+ for key, value in params.config['configurations']['zeppelin-config'].iteritems():
+ my_map[key]=value
+ my_map['zeppelin.server.kerberos.keytab']=params.zeppelin_kerberos_keytab
+ my_map['zeppelin.server.kerberos.principal']=params.zeppelin_kerberos_principal
+
XmlConfig("zeppelin-site.xml",
conf_dir=params.conf_dir,
- configurations=params.config['configurations']['zeppelin-config'],
+ configurations=my_map,
owner=params.zeppelin_user,
group=params.zeppelin_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/d95d3ebe/ambari-server/src/test/python/stacks/2.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json
index 10647fd..fb7cb49 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json
@@ -362,6 +362,8 @@
"zeppelin.notebook.s3.bucket": "zeppelin",
"zeppelin.notebook.homescreen.hide": "false",
"zeppelin.server.allowed.origins": "*",
+ "zeppelin.server.kerberos.keytab": "",
+ "zeppelin.server.kerberos.principal": "",
"zeppelin.interpreter.config.upgrade": "true"
},
"zoo.cfg": {
[17/51] [abbrv] ambari git commit: AMBARI-22665 - Livy2 Does Not
Start On HDP 2.6.0 to 2.6.3 (jonathanhurley)
Posted by am...@apache.org.
AMBARI-22665 - Livy2 Does Not Start On HDP 2.6.0 to 2.6.3 (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5c95c4a7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5c95c4a7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5c95c4a7
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 5c95c4a7a9c75f0b0932b3accf4fb536a3175c0e
Parents: 2817ad4
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Dec 18 12:23:06 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Dec 18 15:53:26 2017 -0500
----------------------------------------------------------------------
.../resource_management/libraries/functions/conf_select.py | 7 +++++++
.../main/resources/custom_actions/scripts/install_packages.py | 4 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c95c4a7/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index d38e273..9de2f8f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -211,6 +211,13 @@ def convert_conf_directories_to_symlinks(package, version, dirs):
# determine which directories would be created, if any are needed
dry_run_directory = create(stack_name, package, version, dry_run = True)
+ # if the dry run reported an error, then we must assume that the package does not exist in
+ # the conf-select tool
+ if len(dry_run_directory) == 0:
+ Logger.info("The conf-select tool reported an error for the package {0}. The configuration linking will be skipped.".format(package))
+ return
+
+
need_dirs = []
for d in dry_run_directory:
if not os.path.exists(d):
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c95c4a7/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index c3fbc48..4cdb0e5 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -184,10 +184,10 @@ class InstallPackages(Script):
from resource_management.libraries.functions import stack_select
package_dirs = conf_select.get_package_dirs()
if package_name in package_dirs:
- Logger.info("Attempting to fix the default conf links for {0}".format(package_name))
+ Logger.info("Determining if the default conf links for {0} need to be fixed".format(package_name))
directories = package_dirs[package_name]
- Logger.info("The following directories will be fixed for {0}: {1}".format(package_name,
+ Logger.info("The following directories will be checked for {0}: {1}".format(package_name,
str(directories)))
stack_version = stack_select.get_stack_version_before_install(component_name)
[45/51] [abbrv] ambari git commit: AMBARI-22634. Kerberos support for
OneFS (amagyar)
Posted by am...@apache.org.
AMBARI-22634. Kerberos support for OneFS (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d4c47772
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d4c47772
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d4c47772
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: d4c47772fb2bf2bdc1370ce29a03e48c8a097cce
Parents: 9378a5f
Author: Attila Magyar <am...@hortonworks.com>
Authored: Thu Dec 14 12:59:07 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../main/admin/kerberos/step1_controller.js | 105 ++++++++++---------
ambari-web/app/messages.js | 1 +
.../app/templates/main/admin/kerberos/step1.hbs | 14 +--
.../addon-services/ONEFS/1.0.0/kerberos.json | 73 +++++++++++++
.../ONEFS/1.0.0/package/scripts/params_linux.py | 12 +--
.../1.0.0/package/scripts/service_check.py | 80 +-------------
6 files changed, 138 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d4c47772/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js b/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js
index 9c864a8..690843b 100644
--- a/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js
+++ b/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js
@@ -18,29 +18,44 @@
var App = require('app');
+var PreCondition = Ember.Object.extend({
+ displayText: null,
+ checked: false,
+ visibilityCriteria: function() { return true; },
+
+ hidden: function() {
+ return !this.get('visibilityCriteria')();
+ }.property('visibilityCriteria'),
+
+ satisfied: function() {
+ return this.get('checked') || this.get('hidden');
+ }.property('checked', 'hidden')
+});
+
App.KerberosWizardStep1Controller = Em.Controller.extend({
name: "kerberosWizardStep1Controller",
selectedItem: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc'),
- isSubmitDisabled: Em.computed.someBy('selectedOption.preConditions', 'checked', false),
+ isSubmitDisabled: Em.computed.someBy('selectedOption.preConditions', 'satisfied', false),
options: Em.A([
Em.Object.create({
displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc'),
value: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc'),
preConditions: [
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.1'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.1')
+ }),
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.2')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.2'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.3')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.3'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.4'),
+ visibilityCriteria: function() { return App.Service.find().someProperty('serviceName', 'ONEFS') }
})
]
}),
@@ -48,25 +63,20 @@ App.KerberosWizardStep1Controller = Em.Controller.extend({
displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.ad'),
value: Em.I18n.t('admin.kerberos.wizard.step1.option.ad'),
preConditions: [
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.1'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.1')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.2'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.2')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.3'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.3')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.4'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.4')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.5'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.5')
})
]
}),
@@ -74,21 +84,17 @@ App.KerberosWizardStep1Controller = Em.Controller.extend({
displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa'),
value: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa'),
preConditions: [
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.1'),
- checked: false
+ PreCondition.create({
+ dsplayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.1'),
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.2'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.2')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.3'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.3')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.4'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.4')
})
]
}),
@@ -96,25 +102,20 @@ App.KerberosWizardStep1Controller = Em.Controller.extend({
displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.manual'),
value: Em.I18n.t('admin.kerberos.wizard.step1.option.manual'),
preConditions: [
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.1'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.1')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.2'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.2')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.3'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.3')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.4'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.4')
}),
- Em.Object.create({
- displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.5'),
- checked: false
+ PreCondition.create({
+ displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.5')
})
]
})
http://git-wip-us.apache.org/repos/asf/ambari/blob/d4c47772/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index b305196..1345d65 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1266,6 +1266,7 @@ Em.I18n.translations = {
'admin.kerberos.wizard.step1.option.kdc.condition.1': 'Ambari Server and cluster hosts have network access to both the KDC and KDC admin hosts.',
'admin.kerberos.wizard.step1.option.kdc.condition.2': 'KDC administrative credentials are on-hand.',
'admin.kerberos.wizard.step1.option.kdc.condition.3': 'The Java Cryptography Extensions (JCE) have been setup on the Ambari Server host and all hosts in the cluster.',
+ 'admin.kerberos.wizard.step1.option.kdc.condition.4': 'The Isilon administrator has setup all appropriate principals in OneFS',
'admin.kerberos.wizard.step1.option.manual': 'Manage Kerberos principals and keytabs manually',
'admin.kerberos.wizard.step1.option.manual.condition.1': 'Cluster hosts have network access to the KDC',
'admin.kerberos.wizard.step1.option.manual.condition.2': 'Kerberos client utilities (such as kinit) have been installed on every cluster host',
http://git-wip-us.apache.org/repos/asf/ambari/blob/d4c47772/ambari-web/app/templates/main/admin/kerberos/step1.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/admin/kerberos/step1.hbs b/ambari-web/app/templates/main/admin/kerberos/step1.hbs
index 57d0637..df15daf 100644
--- a/ambari-web/app/templates/main/admin/kerberos/step1.hbs
+++ b/ambari-web/app/templates/main/admin/kerberos/step1.hbs
@@ -44,12 +44,14 @@
<h5>{{selectedOption.displayName}}:</h5>
<b>{{t admin.kerberos.wizard.step1.prerequisites.label}}</b> <br/> <br/>
{{#each condition in selectedOption.preConditions}}
- {{view App.CheckboxView
- classNames="checkbox"
- checkedBinding="condition.checked"
- labelClassNames="kerberos-step1-prerequiste-checkboxes"
- labelBinding="condition.displayText"
- }}
+ {{#unless condition.hidden}}
+ {{view App.CheckboxView
+ classNames="checkbox"
+ checkedBinding="condition.checked"
+ labelClassNames="kerberos-step1-prerequiste-checkboxes"
+ labelBinding="condition.displayText"
+ }}
+ {{/unless}}
{{/each}}
</div>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d4c47772/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json
new file mode 100644
index 0000000..6078985
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json
@@ -0,0 +1,73 @@
+{
+ "services": [
+ {
+ "name": "ONEFS",
+ "identities": [
+ {
+ "name": "hdfs_spnego",
+ "reference": "/spnego",
+ "principal": {
+ "configuration": "hdfs-site/dfs.web.authentication.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "hdfs-site/dfs.web.authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "hdfs_smokeuser",
+ "reference": "/smokeuser"
+ }
+ ],
+ "auth_to_local_properties" : [
+ "core-site/hadoop.security.auth_to_local"
+ ],
+ "configurations": [
+ {
+ "core-site": {
+ "hadoop.security.authentication": "kerberos",
+ "hadoop.security.authorization": "true",
+ "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}"
+ }
+ },
+ {
+ "ranger-hdfs-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "ONEFS_CLIENT",
+ "identities": [
+ {
+ "name": "hdfs_hdfs_client_hdfs",
+ "principal": {
+ "value": "${hadoop-env/hdfs_user}${principal_suffix}@${realm}",
+ "type" : "user" ,
+ "configuration": "hadoop-env/hdfs_principal_name",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hdfs.headless.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hadoop-env/hdfs_user_keytab"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d4c47772/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
index b823fc5..953efdc 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
@@ -32,24 +32,16 @@ config = Script.get_config()
hostname = config["hostname"]
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None)
user_group = config['configurations']['cluster-env']['user_group']
hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
security_enabled = config['configurations']['cluster-env']['security_enabled']
-namenode_host = default("/clusterHostInfo/namenode_host", [])
-journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", [])
-zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", [])
-
-has_journalnode_hosts = not len(journalnode_hosts) == 0
-has_zkfc_hosts = not len(zkfc_hosts) == 0
-is_namenode_master = hostname in namenode_host
-
dfs_type = default("/commandParams/dfs_type", "")
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None)
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
http://git-wip-us.apache.org/repos/asf/ambari/blob/d4c47772/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
index 3d798a3..4c92866 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py
@@ -18,11 +18,7 @@ limitations under the License.
"""
from resource_management import *
-from resource_management.core.shell import as_user
from ambari_commons.os_family_impl import OsFamilyImpl
-from ambari_commons import OSConst
-from resource_management.libraries.functions.curl_krb_request import curl_krb_request
-from resource_management.core.logger import Logger
class HdfsServiceCheck(Script):
pass
@@ -38,7 +34,7 @@ class HdfsServiceCheckDefault(HdfsServiceCheck):
tmp_file = format("{dir}/{unique}")
if params.security_enabled:
- Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
+ Execute(format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}"),
user=params.hdfs_user
)
params.HdfsResource(dir,
@@ -58,80 +54,6 @@ class HdfsServiceCheckDefault(HdfsServiceCheck):
)
params.HdfsResource(None, action="execute")
- if params.has_journalnode_hosts:
- if params.security_enabled:
- for host in params.journalnode_hosts:
- if params.https_only:
- uri = format("https://{host}:{journalnode_port}")
- else:
- uri = format("http://{host}:{journalnode_port}")
- response, errmsg, time_millis = curl_krb_request(params.tmp_dir, params.smoke_user_keytab,
- params.smokeuser_principal, uri, "jn_service_check",
- params.kinit_path_local, False, None, params.smoke_user)
- if not response:
- Logger.error("Cannot access WEB UI on: {0}. Error : {1}", uri, errmsg)
- return 1
- else:
- journalnode_port = params.journalnode_port
- checkWebUIFileName = "checkWebUI.py"
- checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
- comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
-
- checkWebUICmd = format("ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only} -o {script_https_protocol}")
- File(checkWebUIFilePath,
- content=StaticFile(checkWebUIFileName),
- mode=0775)
-
- Execute(checkWebUICmd,
- logoutput=True,
- try_sleep=3,
- tries=5,
- user=params.smoke_user
- )
-
- if params.is_namenode_master:
- if params.has_zkfc_hosts:
- pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
- pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid")
- check_zkfc_process_cmd = as_user(format(
- "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.hdfs_user)
- Execute(check_zkfc_process_cmd,
- logoutput=True,
- try_sleep=3,
- tries=5
- )
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HdfsServiceCheckWindows(HdfsServiceCheck):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- unique = functions.get_unique_id_and_date()
-
- #Hadoop uses POSIX-style paths, separator is always /
- dir = params.hdfs_tmp_dir
- tmp_file = dir + '/' + unique
-
- #commands for execution
- hadoop_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hadoop.cmd"))
- create_dir_cmd = "%s fs -mkdir %s" % (hadoop_cmd, dir)
- own_dir = "%s fs -chmod 777 %s" % (hadoop_cmd, dir)
- test_dir_exists = "%s fs -test -e %s" % (hadoop_cmd, dir)
- cleanup_cmd = "%s fs -rm %s" % (hadoop_cmd, tmp_file)
- create_file_cmd = "%s fs -put %s %s" % (hadoop_cmd, os.path.join(params.hadoop_conf_dir, "core-site.xml"), tmp_file)
- test_cmd = "%s fs -test -e %s" % (hadoop_cmd, tmp_file)
-
- hdfs_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hdfs.cmd"))
- safemode_command = "%s dfsadmin -safemode get | %s OFF" % (hdfs_cmd, params.grep_exe)
-
- Execute(safemode_command, logoutput=True, try_sleep=3, tries=20)
- Execute(create_dir_cmd, user=params.hdfs_user,logoutput=True, ignore_failures=True)
- Execute(own_dir, user=params.hdfs_user,logoutput=True)
- Execute(test_dir_exists, user=params.hdfs_user,logoutput=True)
- Execute(create_file_cmd, user=params.hdfs_user,logoutput=True)
- Execute(test_cmd, user=params.hdfs_user,logoutput=True)
- Execute(cleanup_cmd, user=params.hdfs_user,logoutput=True)
if __name__ == "__main__":
HdfsServiceCheck().execute()
[51/51] [abbrv] ambari git commit: AMBARI-22674. ServiceAdvisor for
OneFS (amagyar)
Posted by am...@apache.org.
AMBARI-22674. ServiceAdvisor for OneFS (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b32efeca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b32efeca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b32efeca
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: b32efecaf6bf6f3f7bed16b9df194dfc3403e069
Parents: d4c4777
Author: Attila Magyar <am...@hortonworks.com>
Authored: Fri Dec 22 09:54:10 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:58:00 2017 +0100
----------------------------------------------------------------------
.../python/ambari_agent/alerts/ams_alert.py | 4 +-
.../python/ambari_agent/alerts/base_alert.py | 47 +------
.../python/ambari_agent/alerts/metric_alert.py | 3 +-
.../python/ambari_agent/alerts/port_alert.py | 4 +-
.../python/ambari_agent/alerts/web_alert.py | 6 +-
.../main/python/ambari_commons/inet_utils.py | 46 +++++++
.../management-packs/isilon-onefs-mpack/pom.xml | 23 ++++
.../ONEFS/1.0.0/configuration/core-site.xml | 14 +-
.../ONEFS/1.0.0/configuration/hdfs-site.xml | 24 ++--
.../ONEFS/1.0.0/configuration/onefs.xml | 30 +++++
.../addon-services/ONEFS/1.0.0/metainfo.xml | 13 ++
.../ONEFS/1.0.0/package/scripts/params_linux.py | 12 +-
.../ONEFS/1.0.0/service_advisor.py | 122 +++++++++++++++++
.../ONEFS/1.0.0/themes/theme.json | 60 +++++++++
.../src/test/python/TestServiceAdvisor.py | 67 ++++++++++
.../src/test/python/unitTests.py | 132 +++++++++++++++++++
16 files changed, 529 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
index 29b885b..cc2884e 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
@@ -22,7 +22,6 @@ import httplib
import imp
import time
import urllib
-from alerts.base_alert import BaseAlert
from alerts.metric_alert import MetricAlert
import ambari_simplejson as json
import logging
@@ -30,6 +29,7 @@ import re
import uuid
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
+from ambari_commons import inet_utils
logger = logging.getLogger()
@@ -68,7 +68,7 @@ class AmsAlert(MetricAlert):
logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
- host = BaseAlert.get_host_from_url(alert_uri.uri)
+ host = inet_utils.get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
index 5c0305e..cdbe316 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
@@ -510,49 +510,4 @@ class BaseAlert(object):
:param state: the state of the alert in uppercase (such as OK, WARNING, etc)
:return: the parameterized text
'''
- return '{0}'
-
- """
- See RFC3986, Appendix B
- Tested on the following cases:
- "192.168.54.1"
- "192.168.54.2:7661
- "hdfs://192.168.54.3/foo/bar"
- "ftp://192.168.54.4:7842/foo/bar"
-
- Returns None if only a port is passed in
- """
- @staticmethod
- def get_host_from_url(uri):
- if uri is None:
- return None
-
- # if not a string, return None
- if not isinstance(uri, basestring):
- return None
-
- # RFC3986, Appendix B
- parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
-
- # index of parts
- # scheme = 1
- # authority = 3
- # path = 4
- # query = 6
- # fragment = 8
-
- host_and_port = uri
- if 0 == len(parts[0][1]):
- host_and_port = parts[0][4]
- elif 0 == len(parts[0][2]):
- host_and_port = parts[0][1]
- elif parts[0][2].startswith("//"):
- host_and_port = parts[0][3]
-
- if -1 == host_and_port.find(':'):
- if host_and_port.isdigit():
- return None
-
- return host_and_port
- else:
- return host_and_port.split(':')[0]
+ return '{0}'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index 66a1d05..64a1a29 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
from ambari_commons.urllib_handlers import RefreshHeaderProcessor
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_commons import inet_utils
from ambari_agent import Constants
logger = logging.getLogger()
@@ -80,7 +81,7 @@ class MetricAlert(BaseAlert):
logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
- host = BaseAlert.get_host_from_url(alert_uri.uri)
+ host = inet_utils.get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
index 02cc91c..1e2e581 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
@@ -24,7 +24,7 @@ import time
from alerts.base_alert import BaseAlert
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address
+from ambari_commons.inet_utils import resolve_address, get_host_from_url
logger = logging.getLogger()
# default timeouts
@@ -111,7 +111,7 @@ class PortAlert(BaseAlert):
break
- host = BaseAlert.get_host_from_url(uri_value)
+ host = get_host_from_url(uri_value)
if host is None or host == "localhost" or host == "0.0.0.0":
host = self.host_name
host_not_specified = True
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
index 0e400f7..6e01b89 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
@@ -21,9 +21,7 @@ limitations under the License.
import logging
import time
import urllib2
-import ssl
-from functools import wraps
from urllib2 import HTTPError
from tempfile import gettempdir
@@ -33,7 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
from resource_management.libraries.functions.get_path_from_url import get_path_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol
+from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol, get_host_from_url
from ambari_agent import Constants
from ambari_agent.AmbariConfig import AmbariConfig
@@ -135,7 +133,7 @@ class WebAlert(BaseAlert):
uri_path = get_path_from_url(string_uri)
# start building the URL manually
- host = BaseAlert.get_host_from_url(alert_uri.uri)
+ host = get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/ambari-common/src/main/python/ambari_commons/inet_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 5fd3e88..a226235 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -23,6 +23,7 @@ import time
import sys
import urllib2
import socket
+import re
from ambari_commons import OSCheck
from functools import wraps
@@ -266,3 +267,48 @@ def ensure_ssl_using_protocol(protocol="PROTOCOL_TLSv1", ca_certs=None):
return context
_create_default_https_context_patched._ambari_patched = True
ssl._create_default_https_context = _create_default_https_context_patched
+
+"""
+See RFC3986, Appendix B
+Tested on the following cases:
+ "192.168.54.1"
+ "192.168.54.2:7661
+ "hdfs://192.168.54.3/foo/bar"
+ "ftp://192.168.54.4:7842/foo/bar"
+
+ Returns None if only a port is passed in
+"""
+def get_host_from_url(uri):
+ if uri is None:
+ return None
+
+ # if not a string, return None
+ if not isinstance(uri, basestring):
+ return None
+
+ # RFC3986, Appendix B
+ parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
+
+ # index of parts
+ # scheme = 1
+ # authority = 3
+ # path = 4
+ # query = 6
+ # fragment = 8
+
+ host_and_port = uri
+ if 0 == len(parts[0][1]):
+ host_and_port = parts[0][4]
+ elif 0 == len(parts[0][2]):
+ host_and_port = parts[0][1]
+ elif parts[0][2].startswith("//"):
+ host_and_port = parts[0][3]
+
+ if -1 == host_and_port.find(':'):
+ if host_and_port.isdigit():
+ return None
+
+ return host_and_port
+ else:
+ return host_and_port.split(':')[0]
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/pom.xml b/contrib/management-packs/isilon-onefs-mpack/pom.xml
index 5d8f215..6d5411c 100644
--- a/contrib/management-packs/isilon-onefs-mpack/pom.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/pom.xml
@@ -27,6 +27,7 @@
<minAmbariVersion>3.0.0.0</minAmbariVersion>
<maxAmbariVersion></maxAmbariVersion>
<nifiversion>1.0.0</nifiversion>
+ <custom.tests>false</custom.tests>
</properties>
<parent>
<groupId>org.apache.ambari.contrib.mpacks</groupId>
@@ -88,6 +89,28 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <configuration>
+ <executable>python</executable>
+ <workingDirectory>src/test/python</workingDirectory>
+ <arguments>
+ <argument>unitTests.py</argument>
+ <argument>${custom.tests}</argument>
+ </arguments>
+ </configuration>
+ <id>python-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<tarLongFileMode>gnu</tarLongFileMode>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
index 7d3acd7..e2a7922 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
@@ -27,6 +27,12 @@
literal string "local" or a host:port for HDFS.</description>
<final>true</final>
<on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>onefs</type>
+ <name>onefs_host</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>hadoop.security.authentication</name>
@@ -91,12 +97,4 @@ DEFAULT
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
- <property>
- <name>net.topology.script.file.name</name>
- <value>/etc/hadoop/conf/topology_script.py</value>
- <description>
- Location of topology script used by Hadoop to determine the rack location of nodes.
- </description>
- <on-ambari-upgrade add="true"/>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
index cb6544f..727f450 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
@@ -31,26 +31,30 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:8082</value>
- <description>
- The datanode http server address and port.
- </description>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
<name>dfs.namenode.http-address</name>
<value>localhost:8082</value>
<description>The name of the default file system. Either the
literal string "local" or a host:port for HDFS.</description>
<final>true</final>
- <on-ambari-upgrade add="false"/>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>onefs</type>
+ <name>onefs_host</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>dfs.namenode.https-address</name>
<value>localhost:8080</value>
<description>The https address where namenode binds</description>
- <on-ambari-upgrade add="false"/>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>onefs</type>
+ <name>onefs_host</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>dfs.client-write-packet-size</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml
new file mode 100644
index 0000000..bcbf2c8
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>onefs_host</name>
+ <value/>
+ <description>SmartConnect Zone Address of OneFS</description>
+ <display-name>OneFS Host</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
index 5ba562e..06a90d9 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
@@ -41,6 +41,11 @@
<configFiles>
<configFile>
<type>xml</type>
+ <fileName>onefs.xml</fileName>
+ <dictionaryName>onefs</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>xml</type>
<fileName>hdfs-site.xml</fileName>
<dictionaryName>hdfs-site</dictionaryName>
</configFile>
@@ -80,9 +85,17 @@
<timeout>300</timeout>
</commandScript>
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
<configuration-dependencies>
<config-type>core-site</config-type>
<config-type>hdfs-site</config-type>
+ <config-type>onefs</config-type>
<config-type>hadoop-env</config-type>
</configuration-dependencies>
<restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
index 953efdc..79aff8f 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
@@ -17,15 +17,14 @@ limitations under the License.
"""
-from resource_management.libraries.functions.version import format_stack_version, compare_versions
-from resource_management import *
-import os
-import itertools
-import re
+from resource_management import Script
from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format
config = Script.get_config()
@@ -45,6 +44,9 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
+java64_home = config['hostLevelParams']['java_home']
+java_exec = format("{java64_home}/bin/java")
+
import functools
#create partial functions with common arguments for every HdfsResource call
#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py
new file mode 100644
index 0000000..630ff62
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+from ambari_commons import inet_utils
+import imp
+import traceback
+
+def error(message): return {"level": "ERROR", "message": message}
+
+class Uri:
+ @classmethod
+ def default_fs(self, configs):
+ return self.from_config(configs, 'core-site', 'fs.defaultFS')
+
+ @classmethod
+ def http_namenode(self, configs):
+ return self.from_config(configs, 'hdfs-site', 'dfs.namenode.http-address')
+
+ @classmethod
+ def https_namenode(self, configs):
+ return self.from_config(configs, 'hdfs-site', 'dfs.namenode.https-address')
+
+ @classmethod
+ def onefs(self, configs):
+ return self.from_config(configs, 'onefs', 'onefs_host')
+
+ @staticmethod
+ def from_config(configs, config_type, property_name):
+ return Uri(configs['configurations'][config_type]['properties'][property_name])
+
+ def __init__(self, address):
+ self.address = address
+
+ def has_host(self, uri):
+ return uri.hostname() == self.hostname()
+
+ def hostname(self):
+ return inet_utils.get_host_from_url(self.address)
+
+ def fix_host(self, uri):
+ if not uri.hostname() or not self.hostname():
+ return self.address
+ return self.address.replace(self.hostname(), uri.hostname())
+
+ def __str__(self):
+ return self.address
+
+class CoreSite:
+ def __init__(self, configs):
+ self.configs = configs
+
+ def validate(self):
+ invalid_configs = []
+ onefs_host = Uri.onefs(self.configs)
+ if not Uri.default_fs(self.configs).has_host(onefs_host):
+ invalid_configs.append({
+ 'config-name': 'fs.defaultFS',
+ 'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+ })
+ return invalid_configs
+
+class HdfsSite:
+ def __init__(self, configs):
+ self.configs = configs
+
+ def validate(self):
+ invalid_configs = []
+ onefs_host = Uri.onefs(self.configs)
+ if not Uri.http_namenode(self.configs).has_host(onefs_host):
+ invalid_configs.append({
+ 'config-name': 'dfs.namenode.http-address',
+ 'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+ })
+ if not Uri.https_namenode(self.configs).has_host(onefs_host):
+ invalid_configs.append({
+ 'config-name': 'dfs.namenode.https-address',
+ 'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+ })
+ return invalid_configs
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+else:
+ class HDP26ONEFSServiceAdvisor(service_advisor.ServiceAdvisor):
+ def getServiceConfigurationRecommendations(self, configs, clusterData, services, hosts):
+ putCoreSiteProperty = self.putProperty(configs, "core-site", services)
+ putHdfsSiteProperty = self.putProperty(configs, "hdfs-site", services)
+ onefs_host = Uri.onefs(services)
+ putCoreSiteProperty("fs.defaultFS", Uri.default_fs(services).fix_host(onefs_host))
+ putHdfsSiteProperty("dfs.namenode.http-address", Uri.http_namenode(services).fix_host(onefs_host))
+ putHdfsSiteProperty("dfs.namenode.https-address", Uri.https_namenode(services).fix_host(onefs_host))
+
+ def getServiceConfigurationsValidationItems(self, configs, recommendedDefaults, services, hosts):
+ validation_errors = []
+ validation_errors.extend(self.toConfigurationValidationProblems(CoreSite(services).validate(), 'core-site'))
+ validation_errors.extend(self.toConfigurationValidationProblems(HdfsSite(services).validate(), 'hdfs-site'))
+ return validation_errors
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json
new file mode 100644
index 0000000..8301bf0
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json
@@ -0,0 +1,60 @@
+{
+ "name": "default",
+ "description": "Default theme for OneFS service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-onefs",
+ "display-name": "OneFS",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-onefs-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "onefs/onefs_host",
+ "subsection-name": "subsection-onefs-col1"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "onefs/onefs_host",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py b/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py
new file mode 100644
index 0000000..26714b3
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from unittest import TestCase
+from service_advisor import Uri, CoreSite, HdfsSite
+
+configs = {
+ 'configurations': {
+ 'core-site': {
+ 'properties': {
+ 'fs.defaultFS': 'hdfs://localhost:8020'
+ }
+ },
+ 'hdfs-site': {
+ 'properties': {
+ 'dfs.namenode.http-address' : 'scisilon.fqdn:8082',
+ 'dfs.namenode.https-address' : 'scisilon.fqdn:8080',
+ }
+ },
+ 'onefs': {
+ 'properties': {
+ 'onefs_host': 'scisilon.fqdn'
+ }
+ }
+ }
+}
+
+class TestUri(TestCase):
+ def test_fix_host(self):
+ onefs_host = Uri.onefs(configs)
+ default_fs = Uri.default_fs(configs)
+ fixed = default_fs.fix_host(onefs_host)
+ self.assertEquals('hdfs://scisilon.fqdn:8020', fixed)
+
+ def test_skip_replacing_to_empty_host(self):
+ default_fs = Uri.default_fs(configs)
+ self.assertEquals('hdfs://localhost:8020', default_fs.fix_host(Uri("")))
+
+ def test_skip_fixing_invalid_host(self):
+ default_fs = Uri("hdfs://:8080")
+ self.assertEquals('hdfs://:8080', default_fs.fix_host(Uri("host")))
+
+ def test_core_site_validation_error_on_host_mismatch(self):
+ core_site = CoreSite(configs)
+ erros = core_site.validate()
+ self.assertEquals(len(erros), 1)
+ self.assertEquals(erros[0]['config-name'], 'fs.defaultFS')
+
+ def test_hdfs_site_no_validation_error(self):
+ hdfs_site = HdfsSite(configs)
+ erros = hdfs_site.validate()
+ self.assertEquals(len(erros), 0)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b32efeca/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py b/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py
new file mode 100644
index 0000000..03bc923
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import unittest
+import os
+import sys
+from random import shuffle
+import fnmatch
+
+#excluded directories with non-test staff from stack and service scanning,
+#also we can add service or stack to skip here
+STACK_EXCLUDE = ["utils"]
+SERVICE_EXCLUDE = ["configs"]
+
+TEST_MASK = '[Tt]est*.py'
+CUSTOM_TEST_MASK = '_[Tt]est*.py'
+def get_parent_path(base, directory_name):
+ """
+ Returns absolute path for directory_name, if directory_name present in base.
+ For example, base=/home/user/test2, directory_name=user - will return /home/user
+ """
+ done = False
+ while not done:
+ base = os.path.dirname(base)
+ if base == "/":
+ return None
+ if os.path.split(base)[-1] == directory_name:
+ done = True
+ else:
+ done = False
+ return base
+
+def get_test_files(path, mask = None, recursive=True):
+ """
+ Returns test files for path recursively
+ """
+ current = []
+ directory_items = os.listdir(path)
+ directory_items.sort()
+
+ for item in directory_items:
+ add_to_pythonpath = False
+ item_path = os.path.join(path, item)
+ if os.path.isfile(item_path):
+ if fnmatch.fnmatch(item, mask):
+ add_to_pythonpath = True
+ current.append(item)
+ elif os.path.isdir(item_path):
+ if recursive:
+ current.extend(get_test_files(item_path, mask = mask))
+ if add_to_pythonpath:
+ sys.path.append(path)
+ return current
+
+
+def main():
+ custom_tests = False
+ if len(sys.argv) > 1:
+ if sys.argv[1] == "true":
+ custom_tests = True
+ pwd = os.path.abspath(os.path.dirname(__file__))
+
+ project_folder = get_parent_path(pwd,'isilon-onefs-mpack')
+ sys.path.append(project_folder + "/src/main/resources/addon-services/ONEFS/1.0.0")
+ sys.path.append(project_folder + "/src/test/python")
+ sys.path.append(project_folder + "/../../../ambari-agent/src/main/python")
+ sys.path.append(project_folder + "/../../../ambari-common/src/main/python")
+
+ has_failures = False
+ test_runs = 0
+ test_failures = []
+ test_errors = []
+ sys.stderr.write("Running tests\n")
+ if custom_tests:
+ test_mask = CUSTOM_TEST_MASK
+ else:
+ test_mask = TEST_MASK
+
+ tests = get_test_files(pwd, mask=test_mask, recursive=True)
+ shuffle(tests)
+ modules = [os.path.basename(s)[:-3] for s in tests]
+ suites = [unittest.defaultTestLoader.loadTestsFromName(name) for name in
+ modules]
+ testSuite = unittest.TestSuite(suites)
+ textRunner = unittest.TextTestRunner(verbosity=2).run(testSuite)
+ test_runs += textRunner.testsRun
+ test_errors.extend([(str(item[0]),str(item[1]),"ERROR") for item in textRunner.errors])
+ test_failures.extend([(str(item[0]),str(item[1]),"FAIL") for item in textRunner.failures])
+ tests_status = textRunner.wasSuccessful() and not has_failures
+
+ if not tests_status:
+ sys.stderr.write("----------------------------------------------------------------------\n")
+ sys.stderr.write("Failed tests:\n")
+ for failed_tests in [test_errors,test_failures]:
+ for err in failed_tests:
+ sys.stderr.write("{0}: {1}\n".format(err[2],err[0]))
+ sys.stderr.write("----------------------------------------------------------------------\n")
+ sys.stderr.write("{0}\n".format(err[1]))
+ sys.stderr.write("----------------------------------------------------------------------\n")
+ sys.stderr.write("Total run:{0}\n".format(test_runs))
+ sys.stderr.write("Total errors:{0}\n".format(len(test_errors)))
+ sys.stderr.write("Total failures:{0}\n".format(len(test_failures)))
+
+ if tests_status:
+ sys.stderr.write("OK\n")
+ exit_code = 0
+ else:
+ sys.stderr.write("ERROR\n")
+ exit_code = 1
+ return exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+
[03/51] [abbrv] ambari git commit: AMBARI-22639. Log Feeder refactor:
integrate with spring boot (oleewere)
Posted by am...@apache.org.
AMBARI-22639. Log Feeder refactor: integrate with spring boot (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31e8e55a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31e8e55a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31e8e55a
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 31e8e55a900e0dd4915810302da22991e7cc54e4
Parents: a3d4c3d
Author: Oliver Szabo <ol...@gmail.com>
Authored: Tue Dec 12 22:34:09 2017 +0100
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Wed Dec 13 22:44:51 2017 +0100
----------------------------------------------------------------------
.../config/api/LogSearchConfigFactory.java | 49 +-
.../config/api/LogSearchConfigLogFeeder.java | 3 +-
.../ambari-logsearch-logfeeder/.gitignore | 1 +
.../ambari-logsearch-logfeeder/pom.xml | 62 ++-
.../org/apache/ambari/logfeeder/LogFeeder.java | 167 +------
.../ambari/logfeeder/LogFeederCommandLine.java | 28 +-
.../ambari/logfeeder/common/ConfigHandler.java | 47 +-
.../ambari/logfeeder/common/ConfigItem.java | 8 +-
.../logfeeder/common/LogEntryParseTester.java | 3 +-
.../logfeeder/common/LogFeederConstants.java | 50 ++
.../logfeeder/conf/ApplicationConfig.java | 107 ++++
.../logfeeder/conf/InputSimulateConfig.java | 154 ++++++
.../logfeeder/conf/LogEntryCacheConfig.java | 118 +++++
.../ambari/logfeeder/conf/LogFeederProps.java | 226 +++++++++
.../logfeeder/conf/LogFeederSecurityConfig.java | 189 +++++++
.../logfeeder/conf/MetricsCollectorConfig.java | 113 +++++
.../apache/ambari/logfeeder/filter/Filter.java | 7 +-
.../ambari/logfeeder/filter/FilterGrok.java | 7 +-
.../ambari/logfeeder/filter/FilterKeyValue.java | 5 +-
.../logfeeder/input/AbstractInputFile.java | 11 +-
.../apache/ambari/logfeeder/input/Input.java | 25 +-
.../logfeeder/input/InputConfigUploader.java | 82 +--
.../ambari/logfeeder/input/InputManager.java | 28 +-
.../ambari/logfeeder/input/InputSimulate.java | 44 +-
.../logfeeder/loglevelfilter/FilterLogData.java | 73 ---
.../loglevelfilter/LogLevelFilterHandler.java | 89 +++-
.../logfeeder/metrics/LogFeederAMSClient.java | 22 +-
.../logfeeder/metrics/MetricsManager.java | 16 +-
.../ambari/logfeeder/metrics/StatsLogger.java | 83 ++++
.../ambari/logfeeder/output/OutputFile.java | 5 +-
.../ambari/logfeeder/output/OutputHDFSFile.java | 8 +-
.../ambari/logfeeder/output/OutputKafka.java | 5 +-
.../ambari/logfeeder/output/OutputManager.java | 36 +-
.../ambari/logfeeder/output/OutputS3File.java | 8 +-
.../ambari/logfeeder/output/OutputSolr.java | 10 +-
.../logfeeder/util/LogFeederPropertiesUtil.java | 498 -------------------
.../apache/ambari/logfeeder/util/SSLUtil.java | 134 -----
.../shipper-conf/input.config-sample.json | 2 +-
.../src/main/resources/log4j.xml | 15 +-
.../src/main/resources/logfeeder.properties | 18 +-
.../ambari/logfeeder/filter/FilterGrokTest.java | 3 +-
.../ambari/logfeeder/filter/FilterJSONTest.java | 3 +-
.../logfeeder/filter/FilterKeyValueTest.java | 3 +-
.../ambari/logfeeder/input/InputFileTest.java | 11 +-
.../logfeeder/input/InputManagerTest.java | 12 +-
.../logconfig/LogConfigHandlerTest.java | 148 ------
.../logfeeder/metrics/MetricsManagerTest.java | 7 -
.../logfeeder/output/OutputKafkaTest.java | 7 +-
.../logfeeder/output/OutputManagerTest.java | 32 +-
.../logfeeder/output/OutputS3FileTest.java | 9 +-
.../ambari/logfeeder/output/OutputSolrTest.java | 10 +-
51 files changed, 1542 insertions(+), 1259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java
index a84a97b..c74fad3 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java
@@ -37,12 +37,13 @@ public class LogSearchConfigFactory {
* @param properties The properties of the component for which the configuration is created. If the properties contain the
* "logsearch.config.class" entry than the class defined there would be used instead of the default class.
* @param defaultClass The default configuration class to use if not specified otherwise.
+ * @param init initialize the properties and zookeeper client
* @return The Log Search Configuration instance.
* @throws Exception Throws exception if the defined class does not implement LogSearchConfigServer, or doesn't have an empty
* constructor, or throws an exception in it's init method.
*/
public static LogSearchConfigServer createLogSearchConfigServer(Map<String, String> properties,
- Class<? extends LogSearchConfigServer> defaultClass) throws Exception {
+ Class<? extends LogSearchConfigServer> defaultClass, boolean init) throws Exception {
try {
LogSearchConfigServer logSearchConfig = null;
String configClassName = properties.get("logsearch.config.server.class");
@@ -57,8 +58,9 @@ public class LogSearchConfigFactory {
} else {
logSearchConfig = defaultClass.newInstance();
}
-
- logSearchConfig.init(properties);
+ if (init) {
+ logSearchConfig.init(properties);
+ }
return logSearchConfig;
} catch (Exception e) {
LOG.error("Could not initialize logsearch config.", e);
@@ -74,12 +76,13 @@ public class LogSearchConfigFactory {
* "logsearch.config.class" entry than the class defined there would be used instead of the default class.
* @param clusterName The name of the cluster.
* @param defaultClass The default configuration class to use if not specified otherwise.
+ * @param init initialize the properties and zookeeper client
* @return The Log Search Configuration instance.
* @throws Exception Throws exception if the defined class does not implement LogSearchConfigLogFeeder, or doesn't have an empty
* constructor, or throws an exception in it's init method.
*/
public static LogSearchConfigLogFeeder createLogSearchConfigLogFeeder(Map<String, String> properties, String clusterName,
- Class<? extends LogSearchConfigLogFeeder> defaultClass) throws Exception {
+ Class<? extends LogSearchConfigLogFeeder> defaultClass, boolean init) throws Exception {
try {
LogSearchConfigLogFeeder logSearchConfig = null;
String configClassName = properties.get("logsearch.config.logfeeder.class");
@@ -94,12 +97,46 @@ public class LogSearchConfigFactory {
} else {
logSearchConfig = defaultClass.newInstance();
}
-
- logSearchConfig.init(properties, clusterName);
+ if (init) {
+ logSearchConfig.init(properties, clusterName);
+ }
return logSearchConfig;
} catch (Exception e) {
LOG.error("Could not initialize logsearch config.", e);
throw e;
}
}
+
+ /**
+ * Creates a Log Search Configuration instance for the Log Search Server that implements
+ * {@link org.apache.ambari.logsearch.config.api.LogSearchConfigServer}.
+ *
+ * @param properties The properties of the component for which the configuration is created. If the properties contain the
+ * "logsearch.config.class" entry than the class defined there would be used instead of the default class.
+ * @param defaultClass The default configuration class to use if not specified otherwise.
+ * @return The Log Search Configuration instance.
+ * @throws Exception Throws exception if the defined class does not implement LogSearchConfigServer, or doesn't have an empty
+ * constructor, or throws an exception in it's init method.
+ */
+ public static LogSearchConfigServer createLogSearchConfigServer(Map<String, String> properties,
+ Class<? extends LogSearchConfigServer> defaultClass) throws Exception {
+ return createLogSearchConfigServer(properties, defaultClass, true);
+ }
+
+ /**
+ * Creates a Log Search Configuration instance for the Log Search Server that implements
+ * {@link org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder}.
+ *
+ * @param properties The properties of the component for which the configuration is created. If the properties contain the
+ * "logsearch.config.class" entry than the class defined there would be used instead of the default class.
+ * @param clusterName The name of the cluster.
+ * @param defaultClass The default configuration class to use if not specified otherwise.
+ * @return The Log Search Configuration instance.
+ * @throws Exception Throws exception if the defined class does not implement LogSearchConfigLogFeeder, or doesn't have an empty
+ * constructor, or throws an exception in it's init method.
+ */
+ public static LogSearchConfigLogFeeder createLogSearchConfigLogFeeder(Map<String, String> properties, String clusterName,
+ Class<? extends LogSearchConfigLogFeeder> defaultClass) throws Exception {
+ return createLogSearchConfigLogFeeder(properties, clusterName, defaultClass, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
index 6ed36fd..1387515 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigLogFeeder.java
@@ -68,8 +68,7 @@ public interface LogSearchConfigLogFeeder extends LogSearchConfig {
/**
* Saves the properties of an Output Solr.
- *
- * @param type The type of the Output Solr.
+ *
* @param outputConfigMonitors The monitors which want to watch the output config changes.
* @throws Exception
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/.gitignore
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/.gitignore b/ambari-logsearch/ambari-logsearch-logfeeder/.gitignore
new file mode 100644
index 0000000..7b00482
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/.gitignore
@@ -0,0 +1 @@
+*.pid
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index 01710bf..005af15 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -33,6 +33,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <spring.version>4.3.10.RELEASE</spring.version>
+ <spring-boot.version>1.5.6.RELEASE</spring-boot.version>
</properties>
<dependencies>
@@ -67,11 +69,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.17</version>
- </dependency>
- <dependency>
<groupId>io.thekraken</groupId>
<artifactId>grok</artifactId>
<version>0.1.4</version>
@@ -102,16 +99,6 @@
<version>18.0</version>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.20</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>1.7.20</version>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.2</version>
@@ -152,6 +139,10 @@
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</exclusion>
@@ -179,6 +170,32 @@
<artifactId>commons-io</artifactId>
<version>${common.io.version}</version>
</dependency>
+ <dependency>
+ <groupId>javax.inject</groupId>
+ <artifactId>javax.inject</artifactId>
+ <version>1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter</artifactId>
+ <version>${spring-boot.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-log4j</artifactId>
+ <version>1.3.8.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-configuration-processor</artifactId>
+ <version>${spring-boot.version}</version>
+ </dependency>
</dependencies>
<build>
<finalName>LogFeeder</finalName>
@@ -225,6 +242,21 @@
</arguments>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring-boot.version}</version>
+ <configuration>
+ <classifier>exec</classifier>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<!-- copy-dependencies -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
index 5114743..2d31e5a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
@@ -16,146 +16,41 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.ambari.logfeeder;
+import com.google.gson.GsonBuilder;
+import org.apache.ambari.logfeeder.common.LogEntryParseTester;
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.LogManager;
+import org.springframework.boot.Banner;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.system.ApplicationPidFileWriter;
+
import java.io.File;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.ambari.logfeeder.common.ConfigHandler;
-import org.apache.ambari.logfeeder.common.LogEntryParseTester;
-import org.apache.ambari.logsearch.config.api.LogSearchConfigFactory;
-import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
-import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigLogFeederZK;
-import org.apache.commons.io.FileUtils;
-import org.apache.ambari.logfeeder.input.InputConfigUploader;
-import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
-import org.apache.ambari.logfeeder.metrics.MetricData;
-import org.apache.ambari.logfeeder.metrics.MetricsManager;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
-import org.apache.ambari.logfeeder.util.SSLUtil;
-import com.google.common.collect.Maps;
-import com.google.gson.GsonBuilder;
-
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-
+@SpringBootApplication(
+ scanBasePackages = {"org.apache.ambari.logfeeder"}
+)
public class LogFeeder {
- private static final Logger LOG = Logger.getLogger(LogFeeder.class);
-
- private static final int LOGFEEDER_SHUTDOWN_HOOK_PRIORITY = 30;
- private static final int CHECKPOINT_CLEAN_INTERVAL_MS = 24 * 60 * 60 * 60 * 1000; // 24 hours
-
- private final LogFeederCommandLine cli;
-
- private ConfigHandler configHandler;
- private LogSearchConfigLogFeeder config;
-
- private MetricsManager metricsManager = new MetricsManager();
-
- private long lastCheckPointCleanedMS = 0;
- private Thread statLoggerThread = null;
-
- private LogFeeder(LogFeederCommandLine cli) {
- this.cli = cli;
- }
-
- public void run() {
- try {
- init();
- monitor();
- } catch (Throwable t) {
- LOG.fatal("Caught exception in main.", t);
- System.exit(1);
- }
- }
-
- private void init() throws Throwable {
- long startTime = System.currentTimeMillis();
-
- SSLUtil.ensureStorePasswords();
-
- config = LogSearchConfigFactory.createLogSearchConfigLogFeeder(Maps.fromProperties(LogFeederPropertiesUtil.getProperties()),
- LogFeederPropertiesUtil.getClusterName(), LogSearchConfigLogFeederZK.class);
- configHandler = new ConfigHandler(config);
- configHandler.init();
- LogLevelFilterHandler.init(config);
- InputConfigUploader.load(config);
- config.monitorInputConfigChanges(configHandler, new LogLevelFilterHandler(), LogFeederPropertiesUtil.getClusterName());
-
- metricsManager.init();
-
- LOG.debug("==============");
-
- long endTime = System.currentTimeMillis();
- LOG.info("Took " + (endTime - startTime) + " ms to initialize");
- }
-
- private class JVMShutdownHook extends Thread {
-
- public void run() {
- try {
- LOG.info("Processing is shutting down.");
-
- configHandler.close();
- config.close();
- logStats();
-
- LOG.info("LogSearch is exiting.");
- } catch (Throwable t) {
- // Ignore
- }
- }
- }
-
- private void monitor() throws Exception {
- JVMShutdownHook logFeederJVMHook = new JVMShutdownHook();
- ShutdownHookManager.get().addShutdownHook(logFeederJVMHook, LOGFEEDER_SHUTDOWN_HOOK_PRIORITY);
-
- statLoggerThread = new Thread("statLogger") {
-
- @Override
- public void run() {
- while (true) {
- try {
- Thread.sleep(30 * 1000);
- } catch (Throwable t) {
- // Ignore
- }
- try {
- logStats();
- } catch (Throwable t) {
- LOG.error("LogStats: Caught exception while logging stats.", t);
- }
-
- if (System.currentTimeMillis() > (lastCheckPointCleanedMS + CHECKPOINT_CLEAN_INTERVAL_MS)) {
- lastCheckPointCleanedMS = System.currentTimeMillis();
- configHandler.cleanCheckPointFiles();
- }
- }
- }
- };
- statLoggerThread.setDaemon(true);
- statLoggerThread.start();
-
- }
-
- private void logStats() {
- configHandler.logStats();
-
- if (metricsManager.isMetricsEnabled()) {
- List<MetricData> metricsList = new ArrayList<MetricData>();
- configHandler.addMetrics(metricsList);
- metricsManager.useMetrics(metricsList);
+ public static void main(String[] args) {
+ LogFeederCommandLine cli = new LogFeederCommandLine(args);
+ if (cli.isTest()) {
+ test(cli);
}
+ String pidFile = System.getenv("PID_FILE") == null ? "logfeeder.pid" : System.getenv("PID_FILE");
+ new SpringApplicationBuilder(LogFeeder.class)
+ .bannerMode(Banner.Mode.OFF)
+ .listeners(new ApplicationPidFileWriter(pidFile))
+ .run(args);
}
- public void test() {
+ private static void test(LogFeederCommandLine cli) {
try {
LogManager.shutdown();
String testLogEntry = cli.getTestLogEntry();
@@ -173,22 +68,4 @@ public class LogFeeder {
e.printStackTrace(System.out);
}
}
-
- public static void main(String[] args) {
- LogFeederCommandLine cli = new LogFeederCommandLine(args);
-
- LogFeeder logFeeder = new LogFeeder(cli);
-
- if (cli.isMonitor()) {
- try {
- LogFeederPropertiesUtil.loadProperties();
- } catch (Throwable t) {
- LOG.warn("Could not load logfeeder properites");
- System.exit(1);
- }
- logFeeder.run();
- } else if (cli.isTest()) {
- logFeeder.test();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
index d996f98..61e7a1e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java
@@ -30,14 +30,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
public class LogFeederCommandLine {
private static final Logger LOG = LoggerFactory.getLogger(LogFeederCommandLine.class);
-
- private static final String MONITOR_COMMAND = "monitor";
private static final String TEST_COMMAND = "test";
private static final String TEST_LOG_ENTRY_OPTION = "test-log-entry";
@@ -60,11 +57,6 @@ public class LogFeederCommandLine {
.desc("Print commands")
.build();
- Option monitorOption = Option.builder("m")
- .longOpt(MONITOR_COMMAND)
- .desc("Monitor log files")
- .build();
-
Option testOption = Option.builder("t")
.longOpt(TEST_COMMAND)
.desc("Test if log entry is parseable")
@@ -95,7 +87,6 @@ public class LogFeederCommandLine {
.build();
options.addOption(helpOption);
- options.addOption(monitorOption);
options.addOption(testOption);
options.addOption(testLogEntryOption);
options.addOption(testShipperConfOption);
@@ -111,21 +102,14 @@ public class LogFeederCommandLine {
System.exit(0);
}
String command = "";
- if (cli.hasOption("m")) {
- command = MONITOR_COMMAND;
- } else if (cli.hasOption("t")) {
+ if (cli.hasOption("t")) {
command = TEST_COMMAND;
validateRequiredOptions(cli, command, testLogEntryOption, testShipperConfOption);
} else {
- List<String> commands = Arrays.asList(MONITOR_COMMAND, TEST_COMMAND);
- helpFormatter.printHelp(COMMAND_LINE_SYNTAX, options);
- LOG.error(String.format("One of the supported commands is required (%s)", StringUtils.join(commands, "|")));
- System.exit(1);
+ LOG.info("Start application in monitor mode ");
}
} catch (Exception e) {
- LOG.error("Error parsing command line parameters", e);
- helpFormatter.printHelp(COMMAND_LINE_SYNTAX, options);
- System.exit(1);
+ LOG.info("Error parsing command line parameters: {}. LogFeeder will be started in monitoring mode.", e.getMessage());
}
}
@@ -142,12 +126,8 @@ public class LogFeederCommandLine {
}
}
- public boolean isMonitor() {
- return cli.hasOption('m');
- }
-
public boolean isTest() {
- return cli.hasOption('t');
+ return cli != null && cli.hasOption('t');
}
public String getTestLogEntry() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
index 243b344..35c0e6a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java
@@ -33,6 +33,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Maps;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputManager;
@@ -47,7 +49,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ambari.logfeeder.util.AliasUtil.AliasType;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logsearch.config.api.InputConfigMonitor;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
@@ -62,16 +63,24 @@ import org.apache.log4j.Logger;
import com.google.gson.reflect.TypeToken;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import javax.inject.Inject;
+
public class ConfigHandler implements InputConfigMonitor {
private static final Logger LOG = Logger.getLogger(ConfigHandler.class);
private final LogSearchConfigLogFeeder logSearchConfig;
-
- private final OutputManager outputManager = new OutputManager();
- private final InputManager inputManager = new InputManager();
+
+ @Inject
+ private InputManager inputManager;
+ @Inject
+ private OutputManager outputManager;
+ @Inject
+ private LogFeederProps logFeederProps;
private final Map<String, Object> globalConfigs = new HashMap<>();
- private final List<String> globalConfigJsons = new ArrayList<String>();
+ private final List<String> globalConfigJsons = new ArrayList<>();
private final List<InputDescriptor> inputConfigList = new ArrayList<>();
private final List<FilterDescriptor> filterConfigList = new ArrayList<>();
@@ -82,9 +91,11 @@ public class ConfigHandler implements InputConfigMonitor {
public ConfigHandler(LogSearchConfigLogFeeder logSearchConfig) {
this.logSearchConfig = logSearchConfig;
}
-
+
+ @PostConstruct
public void init() throws Exception {
loadConfigFiles();
+ logSearchConfig.init(Maps.fromProperties(logFeederProps.getProperties()), logFeederProps.getClusterName());
loadOutputs();
simulateIfNeeded();
@@ -114,7 +125,7 @@ public class ConfigHandler implements InputConfigMonitor {
private List<String> getConfigFiles() {
List<String> configFiles = new ArrayList<>();
- String logFeederConfigFilesProperty = LogFeederPropertiesUtil.getConfigFiles();
+ String logFeederConfigFilesProperty = logFeederProps.getConfigFiles();
LOG.info("logfeeder.config.files=" + logFeederConfigFilesProperty);
if (logFeederConfigFilesProperty != null) {
configFiles.addAll(Arrays.asList(logFeederConfigFilesProperty.split(",")));
@@ -217,7 +228,7 @@ public class ConfigHandler implements InputConfigMonitor {
}
private void simulateIfNeeded() throws Exception {
- int simulatedInputNumber = LogFeederPropertiesUtil.getSimulateInputNumber();
+ int simulatedInputNumber = logFeederProps.getInputSimulateConfig().getSimulateInputNumber();
if (simulatedInputNumber == 0)
return;
@@ -347,18 +358,15 @@ public class ConfigHandler implements InputConfigMonitor {
}
private void sortFilters() {
- Collections.sort(filterConfigList, new Comparator<FilterDescriptor>() {
- @Override
- public int compare(FilterDescriptor o1, FilterDescriptor o2) {
- Integer o1Sort = o1.getSortOrder();
- Integer o2Sort = o2.getSortOrder();
- if (o1Sort == null || o2Sort == null) {
- return 0;
- }
-
- return o1Sort - o2Sort;
+ Collections.sort(filterConfigList, (o1, o2) -> {
+ Integer o1Sort = o1.getSortOrder();
+ Integer o2Sort = o2.getSortOrder();
+ if (o1Sort == null || o2Sort == null) {
+ return 0;
}
- } );
+
+ return o1Sort - o2Sort;
+ });
}
private void assignOutputsToInputs(String serviceName) {
@@ -428,6 +436,7 @@ public class ConfigHandler implements InputConfigMonitor {
outputManager.addMetricsContainers(metricsList);
}
+ @PreDestroy
public void close() {
inputManager.close();
outputManager.close();
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigItem.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigItem.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigItem.java
index 5c20a8e..30bd9fd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigItem.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigItem.java
@@ -21,6 +21,7 @@ package org.apache.ambari.logfeeder.common;
import java.util.List;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.log4j.Logger;
@@ -30,6 +31,7 @@ public abstract class ConfigItem {
protected static final Logger LOG = Logger.getLogger(ConfigBlock.class);
private boolean drain = false;
+ private LogFeederProps logFeederProps;
public MetricData statMetric = new MetricData(getStatMetricName(), false);
public ConfigItem() {
@@ -59,7 +61,8 @@ public abstract class ConfigItem {
/**
* This method needs to be overwritten by deriving classes.
*/
- public void init() throws Exception {
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ this.logFeederProps = logFeederProps;
}
public abstract boolean isEnabled();
@@ -94,4 +97,7 @@ public abstract class ConfigItem {
this.drain = drain;
}
+ public LogFeederProps getLogFeederProps() {
+ return logFeederProps;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
index ec29f69..1a701e1 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.Output;
@@ -76,7 +77,7 @@ public class LogEntryParseTester {
ConfigHandler configHandler = new ConfigHandler(null);
Input input = configHandler.getTestInput(inputConfig, logId);
final Map<String, Object> result = new HashMap<>();
- input.getFirstFilter().init();
+ input.getFirstFilter().init(new LogFeederProps());
input.addOutput(new Output() {
@Override
public void write(String block, InputMarker inputMarker) throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index a7cccc6..b241831 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -38,4 +38,54 @@ public class LogFeederConstants {
public static final String S3_PATH_SEPARATOR = "/";
public static final String IN_MEMORY_TIMESTAMP = "in_memory_timestamp";
+
+ public static final String LOGFEEDER_PROPERTIES_FILE = "logfeeder.properties";
+ public static final String CLUSTER_NAME_PROPERTY = "cluster.name";
+ public static final String TMP_DIR_PROPERTY = "logfeeder.tmp.dir";
+
+ public static final String METRICS_COLLECTOR_PROTOCOL_PROPERTY = "logfeeder.metrics.collector.protocol";
+ public static final String METRICS_COLLECTOR_PORT_PROPERTY = "logfeeder.metrics.collector.port";
+ public static final String METRICS_COLLECTOR_HOSTS_PROPERTY = "logfeeder.metrics.collector.hosts";
+ public static final String METRICS_COLLECTOR_PATH_PROPERTY = "logfeeder.metrics.collector.path";
+
+ public static final String LOG_FILTER_ENABLE_PROPERTY = "logfeeder.log.filter.enable";
+ public static final String INCLUDE_DEFAULT_LEVEL_PROPERTY = "logfeeder.include.default.level";
+
+ public static final String CONFIG_DIR_PROPERTY = "logfeeder.config.dir";
+ public static final String CONFIG_FILES_PROPERTY = "logfeeder.config.files";
+
+ public static final String SIMULATE_INPUT_NUMBER_PROPERTY = "logfeeder.simulate.input_number";
+ public static final int DEFAULT_SIMULATE_INPUT_NUMBER = 0;
+ public static final String SIMULATE_LOG_LEVEL_PROPERTY = "logfeeder.simulate.log_level";
+ public static final String DEFAULT_SIMULATE_LOG_LEVEL = "WARN";
+ public static final String SIMULATE_NUMBER_OF_WORDS_PROPERTY = "logfeeder.simulate.number_of_words";
+ public static final int DEFAULT_SIMULATE_NUMBER_OF_WORDS = 1000;
+ public static final String SIMULATE_MIN_LOG_WORDS_PROPERTY = "logfeeder.simulate.min_log_words";
+ public static final int DEFAULT_SIMULATE_MIN_LOG_WORDS = 5;
+ public static final String SIMULATE_MAX_LOG_WORDS_PROPERTY = "logfeeder.simulate.max_log_words";
+ public static final int DEFAULT_SIMULATE_MAX_LOG_WORDS = 5;
+ public static final String SIMULATE_SLEEP_MILLISECONDS_PROPERTY = "logfeeder.simulate.sleep_milliseconds";
+ public static final int DEFAULT_SIMULATE_SLEEP_MILLISECONDS = 10000;
+ public static final String SIMULATE_LOG_IDS_PROPERTY = "logfeeder.simulate.log_ids";
+
+ public static final String SOLR_KERBEROS_ENABLE_PROPERTY = "logfeeder.solr.kerberos.enable";
+ public static final boolean DEFAULT_SOLR_KERBEROS_ENABLE = false;
+ public static final String DEFAULT_SOLR_JAAS_FILE = "/etc/security/keytabs/logsearch_solr.service.keytab";
+ public static final String SOLR_JAAS_FILE_PROPERTY = "logfeeder.solr.jaas.file";
+
+ public static final String CACHE_ENABLED_PROPERTY = "logfeeder.cache.enabled";
+ public static final boolean DEFAULT_CACHE_ENABLED = false;
+ public static final String CACHE_KEY_FIELD_PROPERTY = "logfeeder.cache.key.field";
+ public static final String DEFAULT_CACHE_KEY_FIELD = "log_message";
+ public static final String CACHE_SIZE_PROPERTY = "logfeeder.cache.size";
+ public static final int DEFAULT_CACHE_SIZE = 100;
+ public static final String CACHE_LAST_DEDUP_ENABLED_PROPERTY = "logfeeder.cache.last.dedup.enabled";
+ public static final boolean DEFAULT_CACHE_LAST_DEDUP_ENABLED = false;
+ public static final String CACHE_DEDUP_INTERVAL_PROPERTY = "logfeeder.cache.dedup.interval";
+ public static final long DEFAULT_CACHE_DEDUP_INTERVAL = 1000;
+
+ public static final String CHECKPOINT_FOLDER_PROPERTY = "logfeeder.checkpoint.folder";
+ public static final String CHECKPOINT_EXTENSION_PROPERTY = "logfeeder.checkpoint.extension";
+ public static final String DEFAULT_CHECKPOINT_EXTENSION = ".cp";
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java
new file mode 100644
index 0000000..cfb6c78
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/ApplicationConfig.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.conf;
+
+import com.google.common.collect.Maps;
+import org.apache.ambari.logfeeder.common.ConfigHandler;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.input.InputConfigUploader;
+import org.apache.ambari.logfeeder.input.InputManager;
+import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
+import org.apache.ambari.logfeeder.metrics.MetricsManager;
+import org.apache.ambari.logfeeder.metrics.StatsLogger;
+import org.apache.ambari.logfeeder.output.OutputManager;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigFactory;
+import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
+import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigLogFeederZK;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.DependsOn;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
+
+import javax.inject.Inject;
+
+@Configuration
+@PropertySource(value = {
+ "classpath:" + LogFeederConstants.LOGFEEDER_PROPERTIES_FILE
+})
+public class ApplicationConfig {
+
+ @Inject
+ private LogFeederProps logFeederProps;
+
+ @Bean
+ public static PropertySourcesPlaceholderConfigurer propertyConfigurer() {
+ return new PropertySourcesPlaceholderConfigurer();
+ }
+
+ @Bean
+ public LogFeederSecurityConfig logFeederSecurityConfig() {
+ return new LogFeederSecurityConfig();
+ }
+
+ @Bean
+ @DependsOn("logSearchConfigLogFeeder")
+ public ConfigHandler configHandler() throws Exception {
+ return new ConfigHandler(logSearchConfigLogFeeder());
+ }
+
+ @Bean
+ @DependsOn("logFeederSecurityConfig")
+ public LogSearchConfigLogFeeder logSearchConfigLogFeeder() throws Exception {
+ return LogSearchConfigFactory.createLogSearchConfigLogFeeder(
+ Maps.fromProperties(logFeederProps.getProperties()),
+ logFeederProps.getClusterName(),
+ LogSearchConfigLogFeederZK.class,false);
+ }
+
+ @Bean
+ public MetricsManager metricsManager() {
+ return new MetricsManager();
+ }
+
+ @Bean
+ @DependsOn("configHandler")
+ public LogLevelFilterHandler logLevelFilterHandler() throws Exception {
+ return new LogLevelFilterHandler(logSearchConfigLogFeeder());
+ }
+
+ @Bean
+ @DependsOn({"configHandler", "logSearchConfigLogFeeder", "logLevelFilterHandler"})
+ public InputConfigUploader inputConfigUploader() {
+ return new InputConfigUploader();
+ }
+
+ @Bean
+ @DependsOn("inputConfigUploader")
+ public StatsLogger statsLogger() {
+ return new StatsLogger();
+ }
+
+ @Bean
+ public InputManager inputManager() {
+ return new InputManager();
+ }
+
+ @Bean
+ public OutputManager outputManager() {
+ return new OutputManager();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/InputSimulateConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/InputSimulateConfig.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/InputSimulateConfig.java
new file mode 100644
index 0000000..cf087f9
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/InputSimulateConfig.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.conf;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class InputSimulateConfig {
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_INPUT_NUMBER_PROPERTY,
+ description = "The number of the simulator instances to run with. O means no simulation.",
+ examples = {"10"},
+ defaultValue = LogFeederConstants.DEFAULT_SIMULATE_INPUT_NUMBER + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.SIMULATE_INPUT_NUMBER_PROPERTY + ":0}")
+ private Integer simulateInputNumber;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_LOG_LEVEL_PROPERTY,
+ description = "The log level to create the simulated log entries with.",
+ examples = {"INFO"},
+ defaultValue = LogFeederConstants.DEFAULT_SIMULATE_LOG_LEVEL,
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.SIMULATE_LOG_LEVEL_PROPERTY + ":"+ LogFeederConstants.DEFAULT_SIMULATE_LOG_LEVEL + "}")
+ private String simulateLogLevel;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_NUMBER_OF_WORDS_PROPERTY,
+ description = "The size of the set of words that may be used to create the simulated log entries with.",
+ examples = {"100"},
+ defaultValue = LogFeederConstants.DEFAULT_SIMULATE_NUMBER_OF_WORDS + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SIMULATE_NUMBER_OF_WORDS_PROPERTY + ":" + LogFeederConstants.DEFAULT_SIMULATE_NUMBER_OF_WORDS + "}")
+ private Integer simulateNumberOfWords;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_MIN_LOG_WORDS_PROPERTY,
+ description = "The minimum number of words in a simulated log entry.",
+ examples = {"3"},
+ defaultValue = LogFeederConstants.DEFAULT_SIMULATE_MIN_LOG_WORDS + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SIMULATE_MIN_LOG_WORDS_PROPERTY + ":" + LogFeederConstants.DEFAULT_SIMULATE_MIN_LOG_WORDS + "}")
+ private Integer simulateMinLogWords;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_MAX_LOG_WORDS_PROPERTY,
+ description = "The maximum number of words in a simulated log entry.",
+ examples = {"8"},
+ defaultValue = LogFeederConstants.DEFAULT_SIMULATE_MAX_LOG_WORDS + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SIMULATE_MAX_LOG_WORDS_PROPERTY + ":" + LogFeederConstants.DEFAULT_SIMULATE_MAX_LOG_WORDS + "}")
+ private Integer simulateMaxLogWords;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_SLEEP_MILLISECONDS_PROPERTY,
+ description = "The milliseconds to sleep between creating two simulated log entries.",
+ examples = {"5000"},
+ defaultValue = LogFeederConstants.DEFAULT_SIMULATE_SLEEP_MILLISECONDS + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SIMULATE_SLEEP_MILLISECONDS_PROPERTY + ":" + LogFeederConstants.DEFAULT_SIMULATE_SLEEP_MILLISECONDS + "}")
+ private Integer simulateSleepMilliseconds;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SIMULATE_LOG_IDS_PROPERTY,
+ description = "The comma separated list of log ids for which to create the simulated log entries.",
+ examples = {"ambari_server,zookeeper,infra_solr,logsearch_app"},
+ defaultValue = "The log ids of the installed services in the cluster",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SIMULATE_LOG_IDS_PROPERTY + ":}")
+ private String simulateLogIds;
+
+ public Integer getSimulateInputNumber() {
+ return simulateInputNumber;
+ }
+
+ public void setSimulateInputNumber(Integer simulateInputNumber) {
+ this.simulateInputNumber = simulateInputNumber;
+ }
+
+ public String getSimulateLogLevel() {
+ return simulateLogLevel;
+ }
+
+ public void setSimulateLogLevel(String simulateLogLevel) {
+ this.simulateLogLevel = simulateLogLevel;
+ }
+
+ public Integer getSimulateNumberOfWords() {
+ return simulateNumberOfWords;
+ }
+
+ public void setSimulateNumberOfWords(Integer simulateNumberOfWords) {
+ this.simulateNumberOfWords = simulateNumberOfWords;
+ }
+
+ public Integer getSimulateMinLogWords() {
+ return simulateMinLogWords;
+ }
+
+ public void setSimulateMinLogWords(Integer simulateMinLogWords) {
+ this.simulateMinLogWords = simulateMinLogWords;
+ }
+
+ public Integer getSimulateMaxLogWords() {
+ return simulateMaxLogWords;
+ }
+
+ public void setSimulateMaxLogWords(Integer simulateMaxLogWords) {
+ this.simulateMaxLogWords = simulateMaxLogWords;
+ }
+
+ public Integer getSimulateSleepMilliseconds() {
+ return simulateSleepMilliseconds;
+ }
+
+ public void setSimulateSleepMilliseconds(Integer simulateSleepMilliseconds) {
+ this.simulateSleepMilliseconds = simulateSleepMilliseconds;
+ }
+
+ public String getSimulateLogIds() {
+ return simulateLogIds;
+ }
+
+ public void setSimulateLogIds(String simulateLogIds) {
+ this.simulateLogIds = simulateLogIds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogEntryCacheConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogEntryCacheConfig.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogEntryCacheConfig.java
new file mode 100644
index 0000000..353bdc1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogEntryCacheConfig.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.conf;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class LogEntryCacheConfig {
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CACHE_ENABLED_PROPERTY,
+ description = "Enables the usage of a cache to avoid duplications.",
+ examples = {"true"},
+ defaultValue = LogFeederConstants.DEFAULT_CACHE_ENABLED + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CACHE_ENABLED_PROPERTY + ":" + LogFeederConstants.DEFAULT_CACHE_ENABLED + "}")
+ private boolean cacheEnabled;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CACHE_KEY_FIELD_PROPERTY,
+ description = "The field which's value should be cached and should be checked for repetitions.",
+ examples = {"some_field_prone_to_repeating_value"},
+ defaultValue = LogFeederConstants.DEFAULT_CACHE_KEY_FIELD,
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CACHE_KEY_FIELD_PROPERTY + ":" + LogFeederConstants.DEFAULT_CACHE_KEY_FIELD + "}")
+ private String cacheKeyField;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CACHE_SIZE_PROPERTY,
+ description = "The number of log entries to cache in order to avoid duplications.",
+ examples = {"50"},
+ defaultValue = LogFeederConstants.DEFAULT_CACHE_SIZE + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CACHE_SIZE_PROPERTY + ":" + LogFeederConstants.DEFAULT_CACHE_SIZE + "}")
+ private Integer cacheSize;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CACHE_LAST_DEDUP_ENABLED_PROPERTY,
+ description = "Enable filtering directly repeating log entries irrelevant of the time spent between them.",
+ examples = {"true"},
+ defaultValue = LogFeederConstants.DEFAULT_CACHE_LAST_DEDUP_ENABLED + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CACHE_LAST_DEDUP_ENABLED_PROPERTY + ":" + LogFeederConstants.DEFAULT_CACHE_LAST_DEDUP_ENABLED + "}")
+ private Boolean cacheLastDedupEnabled;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CACHE_DEDUP_INTERVAL_PROPERTY,
+ description = "Maximum number of milliseconds between two identical messages to be filtered out.",
+ examples = {"500"},
+ defaultValue = LogFeederConstants.DEFAULT_CACHE_DEDUP_INTERVAL + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CACHE_DEDUP_INTERVAL_PROPERTY + ":" + LogFeederConstants.DEFAULT_CACHE_DEDUP_INTERVAL + "}")
+ private String cacheDedupInterval;
+
+ public boolean isCacheEnabled() {
+ return cacheEnabled;
+ }
+
+ public void setCacheEnabled(boolean cacheEnabled) {
+ this.cacheEnabled = cacheEnabled;
+ }
+
+ public String getCacheKeyField() {
+ return cacheKeyField;
+ }
+
+ public void setCacheKeyField(String cacheKeyField) {
+ this.cacheKeyField = cacheKeyField;
+ }
+
+ public Integer getCacheSize() {
+ return cacheSize;
+ }
+
+ public void setCacheSize(Integer cacheSize) {
+ this.cacheSize = cacheSize;
+ }
+
+ public boolean isCacheLastDedupEnabled() {
+ return cacheLastDedupEnabled;
+ }
+
+ public void setCacheLastDedupEnabled(boolean cacheLastDedupEnabled) {
+ this.cacheLastDedupEnabled = cacheLastDedupEnabled;
+ }
+
+ public String getCacheDedupInterval() {
+ return cacheDedupInterval;
+ }
+
+ public void setCacheDedupInterval(String cacheDedupInterval) {
+ this.cacheDedupInterval = cacheDedupInterval;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
new file mode 100644
index 0000000..367d1cd
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.conf;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.AbstractEnvironment;
+import org.springframework.core.env.Environment;
+import org.springframework.core.env.MapPropertySource;
+import org.springframework.core.env.MutablePropertySources;
+import org.springframework.core.io.support.ResourcePropertySource;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.stream.Stream;
+
+@Configuration
+public class LogFeederProps {
+
+ @Inject
+ private Environment env;
+
+ private Properties properties;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CLUSTER_NAME_PROPERTY,
+ description = "The name of the cluster the Log Feeder program runs in.",
+ examples = {"cl1"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CLUSTER_NAME_PROPERTY + "}")
+ private String clusterName;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.TMP_DIR_PROPERTY,
+ description = "The tmp dir used for creating temporary files.",
+ examples = {"/tmp/"},
+ defaultValue = "java.io.tmpdir",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.TMP_DIR_PROPERTY + ":#{systemProperties['java.io.tmpdir']}}")
+ private String tmpDir;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.LOG_FILTER_ENABLE_PROPERTY,
+ description = "Enables the filtering of the log entries by log level filters.",
+ examples = {"true"},
+ defaultValue = "false",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.LOG_FILTER_ENABLE_PROPERTY + "}")
+ private boolean logLevelFilterEnabled;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.INCLUDE_DEFAULT_LEVEL_PROPERTY,
+ description = "Comma separated list of the default log levels to be enabled by the filtering.",
+ examples = {"FATAL,ERROR,WARN"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("#{'${" + LogFeederConstants.INCLUDE_DEFAULT_LEVEL_PROPERTY + ":}'.split(',')}")
+ private List<String> includeDefaultLogLevels;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CONFIG_DIR_PROPERTY,
+ description = "The directory where shipper configuration files are looked for.",
+ examples = {"/etc/ambari-logsearch-logfeeder/conf"},
+ defaultValue = "etc/ambari-logsearch-logfeeder/conf",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.CONFIG_DIR_PROPERTY + ":/etc/ambari-logsearch-logfeeder/conf}")
+ private String confDir;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CONFIG_FILES_PROPERTY,
+ description = "Comma separated list of the config files containing global / output configurations.",
+ examples = {"global.json,output.json", "/etc/ambari-logsearch-logfeeder/conf/global.json"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.CONFIG_FILES_PROPERTY + ":}")
+ private String configFiles;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CHECKPOINT_EXTENSION_PROPERTY,
+ description = "The extension used for checkpoint files.",
+ examples = {"ckp"},
+ defaultValue = LogFeederConstants.DEFAULT_CHECKPOINT_EXTENSION,
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CHECKPOINT_EXTENSION_PROPERTY + ":" + LogFeederConstants.DEFAULT_CHECKPOINT_EXTENSION + "}")
+ private String checkPointExtension;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CHECKPOINT_FOLDER_PROPERTY,
+ description = "The folder where checkpoint files are stored.",
+ examples = {"/etc/ambari-logsearch-logfeeder/conf/checkpoints"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.CHECKPOINT_FOLDER_PROPERTY + ":/etc/ambari-logsearch-logfeeder/conf/checkpoints}")
+ public String checkpointFolder;
+
+ @Inject
+ private LogEntryCacheConfig logEntryCacheConfig;
+
+ @Inject
+ private InputSimulateConfig inputSimulateConfig;
+
+ @Inject
+ private LogFeederSecurityConfig logFeederSecurityConfig;
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public String getTmpDir() {
+ return tmpDir;
+ }
+
+ public boolean isLogLevelFilterEnabled() {
+ return logLevelFilterEnabled;
+ }
+
+ public List<String> getIncludeDefaultLogLevels() {
+ return includeDefaultLogLevels;
+ }
+
+ public String getConfDir() {
+ return confDir;
+ }
+
+ public void setConfDir(String confDir) {
+ this.confDir = confDir;
+ }
+
+ public String getConfigFiles() {
+ return configFiles;
+ }
+
+ public void setConfigFiles(String configFiles) {
+ this.configFiles = configFiles;
+ }
+
+ public LogEntryCacheConfig getLogEntryCacheConfig() {
+ return logEntryCacheConfig;
+ }
+
+ public void setLogEntryCacheConfig(LogEntryCacheConfig logEntryCacheConfig) {
+ this.logEntryCacheConfig = logEntryCacheConfig;
+ }
+
+ public InputSimulateConfig getInputSimulateConfig() {
+ return inputSimulateConfig;
+ }
+
+ public void setInputSimulateConfig(InputSimulateConfig inputSimulateConfig) {
+ this.inputSimulateConfig = inputSimulateConfig;
+ }
+
+ public LogFeederSecurityConfig getLogFeederSecurityConfig() {
+ return logFeederSecurityConfig;
+ }
+
+ public void setLogFeederSecurityConfig(LogFeederSecurityConfig logFeederSecurityConfig) {
+ this.logFeederSecurityConfig = logFeederSecurityConfig;
+ }
+
+ public String getCheckPointExtension() {
+ return checkPointExtension;
+ }
+
+ public void setCheckPointExtension(String checkPointExtension) {
+ this.checkPointExtension = checkPointExtension;
+ }
+
+ public String getCheckpointFolder() {
+ return checkpointFolder;
+ }
+
+ public void setCheckpointFolder(String checkpointFolder) {
+ this.checkpointFolder = checkpointFolder;
+ }
+
+ @PostConstruct
+ public void init() {
+ properties = new Properties();
+ MutablePropertySources propSrcs = ((AbstractEnvironment) env).getPropertySources();
+ ResourcePropertySource propertySource = (ResourcePropertySource) propSrcs.get("class path resource [" +
+ LogFeederConstants.LOGFEEDER_PROPERTIES_FILE + "]");
+ if (propertySource != null) {
+ Stream.of(propertySource)
+ .map(MapPropertySource::getPropertyNames)
+ .flatMap(Arrays::<String>stream)
+ .forEach(propName -> properties.setProperty(propName, env.getProperty(propName)));
+ } else {
+ throw new IllegalArgumentException("Cannot find logfeeder.properties on the classpath");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederSecurityConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederSecurityConfig.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederSecurityConfig.java
new file mode 100644
index 0000000..8a45753
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederSecurityConfig.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.conf;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+import javax.annotation.PostConstruct;
+import java.io.File;
+import java.nio.charset.Charset;
+
+public class LogFeederSecurityConfig {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFeederSecurityConfig.class);
+
+ private static final String KEYSTORE_LOCATION_ARG = "javax.net.ssl.keyStore";
+ private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore";
+ private static final String KEYSTORE_TYPE_ARG = "javax.net.ssl.keyStoreType";
+ private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType";
+ private static final String KEYSTORE_PASSWORD_ARG = "javax.net.ssl.keyStorePassword";
+ private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword";
+ private static final String KEYSTORE_PASSWORD_PROPERTY_NAME = "logfeeder_keystore_password";
+ private static final String TRUSTSTORE_PASSWORD_PROPERTY_NAME = "logfeeder_truststore_password";
+ private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt";
+ private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt";
+
+ private static final String LOGFEEDER_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys";
+ private static final String LOGFEEDER_STORE_DEFAULT_PASSWORD = "bigdata";
+
+ private static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path";
+
+ @LogSearchPropertyDescription(
+ name = CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY,
+ description = "The jceks file that provides passwords.",
+ examples = {"jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder.jceks"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY + ":}")
+ private String credentialStoreProviderPath;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SOLR_JAAS_FILE_PROPERTY,
+ description = "The jaas file used for solr.",
+ examples = {"/etc/ambari-logsearch-logfeeder/conf/logfeeder_jaas.conf"},
+ defaultValue = LogFeederConstants.DEFAULT_SOLR_JAAS_FILE,
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.SOLR_JAAS_FILE_PROPERTY + ":" + LogFeederConstants.DEFAULT_SOLR_JAAS_FILE + "}")
+ private String solrJaasFile;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.SOLR_KERBEROS_ENABLE_PROPERTY,
+ description = "Enables using kerberos for accessing solr.",
+ examples = {"true"},
+ defaultValue = LogFeederConstants.DEFAULT_SOLR_KERBEROS_ENABLE + "",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.SOLR_KERBEROS_ENABLE_PROPERTY + ":" + LogFeederConstants.DEFAULT_SOLR_KERBEROS_ENABLE + "}")
+ private Boolean solrKerberosEnabled;
+
+ public String getKeyStoreLocation() {
+ return System.getProperty(KEYSTORE_LOCATION_ARG);
+ }
+
+ public String getKeyStoreType() {
+ return System.getProperty(KEYSTORE_TYPE_ARG);
+ }
+
+ public String getKeyStorePassword() {
+ return System.getProperty(KEYSTORE_PASSWORD_ARG);
+ }
+
+ public String getTrustStoreLocation() {
+ return System.getProperty(TRUSTSTORE_LOCATION_ARG);
+ }
+
+ public String getTrustStoreType() {
+ return System.getProperty(TRUSTSTORE_TYPE_ARG);
+ }
+
+ public String getTrustStorePassword() {
+ return System.getProperty(TRUSTSTORE_PASSWORD_ARG);
+ }
+
+ public String getCredentialStoreProviderPath() {
+ return credentialStoreProviderPath;
+ }
+
+ public void setCredentialStoreProviderPath(String credentialStoreProviderPath) {
+ this.credentialStoreProviderPath = credentialStoreProviderPath;
+ }
+
+ public String getSolrJaasFile() {
+ return solrJaasFile;
+ }
+
+ public void setSolrJaasFile(String solrJaasFile) {
+ this.solrJaasFile = solrJaasFile;
+ }
+
+ public boolean isSolrKerberosEnabled() {
+ return solrKerberosEnabled;
+ }
+
+ public void setSolrKerberosEnabled(Boolean solrKerberosEnabled) {
+ this.solrKerberosEnabled = solrKerberosEnabled;
+ }
+
+ @PostConstruct
+ public void ensureStorePasswords() {
+ ensureStorePassword(KEYSTORE_LOCATION_ARG, KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_PROPERTY_NAME, KEYSTORE_PASSWORD_FILE);
+ ensureStorePassword(TRUSTSTORE_LOCATION_ARG, TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_PROPERTY_NAME, TRUSTSTORE_PASSWORD_FILE);
+ }
+
+ private void ensureStorePassword(String locationArg, String pwdArg, String propertyName, String fileName) {
+ if (StringUtils.isNotEmpty(System.getProperty(locationArg)) && StringUtils.isEmpty(System.getProperty(pwdArg))) {
+ String password = getPassword(propertyName, fileName);
+ System.setProperty(pwdArg, password);
+ }
+ }
+
+ private String getPassword(String propertyName, String fileName) {
+ String credentialStorePassword = getPasswordFromCredentialStore(propertyName);
+ if (credentialStorePassword != null) {
+ return credentialStorePassword;
+ }
+
+ String filePassword = getPasswordFromFile(fileName);
+ if (filePassword != null) {
+ return filePassword;
+ }
+
+ return LOGFEEDER_STORE_DEFAULT_PASSWORD;
+ }
+
+ private String getPasswordFromCredentialStore(String propertyName) {
+ try {
+ if (StringUtils.isEmpty(credentialStoreProviderPath)) {
+ return null;
+ }
+
+ org.apache.hadoop.conf.Configuration config = new org.apache.hadoop.conf.Configuration();
+ config.set(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, credentialStoreProviderPath);
+ char[] passwordChars = config.getPassword(propertyName);
+ return (ArrayUtils.isNotEmpty(passwordChars)) ? new String(passwordChars) : null;
+ } catch (Exception e) {
+ LOG.warn(String.format("Could not load password %s from credential store, using default password", propertyName));
+ return null;
+ }
+ }
+
+ private String getPasswordFromFile(String fileName) {
+ try {
+ File pwdFile = new File(LOGFEEDER_CERT_DEFAULT_FOLDER, fileName);
+ if (!pwdFile.exists()) {
+ FileUtils.writeStringToFile(pwdFile, LOGFEEDER_STORE_DEFAULT_PASSWORD, Charset.defaultCharset());
+ return LOGFEEDER_STORE_DEFAULT_PASSWORD;
+ } else {
+ return FileUtils.readFileToString(pwdFile, Charset.defaultCharset());
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception occurred during read/write password file for keystore/truststore.", e);
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/MetricsCollectorConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/MetricsCollectorConfig.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/MetricsCollectorConfig.java
new file mode 100644
index 0000000..4b3c6fb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/MetricsCollectorConfig.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.conf;
+
+import com.google.common.base.Splitter;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+
+import javax.annotation.PostConstruct;
+import java.util.List;
+
+@Configuration
+public class MetricsCollectorConfig {
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.METRICS_COLLECTOR_HOSTS_PROPERTY,
+ description = "Comma separtaed list of metric collector hosts.",
+ examples = {"c6401.ambari.apache.org,c6402.ambari.apache.org"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.METRICS_COLLECTOR_HOSTS_PROPERTY + ":}")
+ private String hostsString;
+
+ private List<String> hosts;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.METRICS_COLLECTOR_PROTOCOL_PROPERTY,
+ description = "The protocol used by metric collectors.",
+ examples = {"http", "https"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.METRICS_COLLECTOR_PROTOCOL_PROPERTY + ":#{NULL}}")
+ private String protocol;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.METRICS_COLLECTOR_PORT_PROPERTY,
+ description = "The port used by metric collectors.",
+ examples = {"6188"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.METRICS_COLLECTOR_PORT_PROPERTY + ":#{NULL}}")
+ private String port;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.METRICS_COLLECTOR_PATH_PROPERTY,
+ description = "The path used by metric collectors.",
+ examples = {"/ws/v1/timeline/metrics"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${" + LogFeederConstants.METRICS_COLLECTOR_PATH_PROPERTY + ":#{NULL}}")
+ private String path;
+
+ public List<String> getHosts() {
+ return hosts;
+ }
+
+ public void setHosts(List<String> hosts) {
+ this.hosts = hosts;
+ }
+
+ public String getProtocol() {
+ return protocol;
+ }
+
+ public String getPort() {
+ return port;
+ }
+
+ public void setPort(String port) {
+ this.port = port;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public String getHostsString() {
+ return hostsString;
+ }
+
+ @PostConstruct
+ public void init() {
+ if (StringUtils.isNotBlank(hostsString)) {
+ hosts = Splitter.on(',').splitToList(hostsString);
+ } else {
+ hosts = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
index 8e8834b..a06b348 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.ambari.logfeeder.common.ConfigItem;
import org.apache.ambari.logfeeder.common.LogFeederException;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.mapper.Mapper;
@@ -56,12 +57,12 @@ public abstract class Filter extends ConfigItem {
}
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
initializePostMapValues();
if (nextFilter != null) {
- nextFilter.init();
+ nextFilter.init(logFeederProps);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
index fc7a565..f0ef31b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
@@ -35,6 +35,7 @@ import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.exception.GrokException;
import org.apache.ambari.logfeeder.common.LogFeederException;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
@@ -73,13 +74,13 @@ public class FilterGrok extends Filter {
private MetricData grokErrorMetric = new MetricData("filter.error.grok", false);
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
try {
messagePattern = escapePattern(((FilterGrokDescriptor)filterDescriptor).getMessagePattern());
multilinePattern = escapePattern(((FilterGrokDescriptor)filterDescriptor).getMultilinePattern());
- sourceField = ((FilterGrokDescriptor)filterDescriptor).getSourceField();
+ sourceField = filterDescriptor.getSourceField();
removeSourceField = BooleanUtils.toBooleanDefaultIfNull(filterDescriptor.isRemoveSourceField(), removeSourceField);
LOG.info("init() done. grokPattern=" + messagePattern + ", multilinePattern=" + multilinePattern + ", " +
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
index 8e5aee8..adcf0a4 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.regex.Pattern;
import org.apache.ambari.logfeeder.common.LogFeederException;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
@@ -41,8 +42,8 @@ public class FilterKeyValue extends Filter {
private MetricData errorMetric = new MetricData("filter.error.keyvalue", false);
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
sourceField = filterDescriptor.getSourceField();
valueSplit = StringUtils.defaultString(((FilterKeyValueDescriptor)filterDescriptor).getValueSplit(), valueSplit);
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
index b021c37..cf295c5 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
@@ -28,8 +28,8 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.util.FileUtil;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputFileBaseDescriptor;
import org.apache.commons.lang.ObjectUtils;
@@ -55,6 +55,8 @@ public abstract class AbstractInputFile extends Input {
private Map<String, Map<String, Object>> jsonCheckPoints = new HashMap<>();
private Map<String, InputMarker> lastCheckPointInputMarkers = new HashMap<>();
+ private LogFeederProps logFeederProps;
+
@Override
protected String getStatMetricName() {
return "input.files.read_lines";
@@ -66,10 +68,11 @@ public abstract class AbstractInputFile extends Input {
}
@Override
- public void init() throws Exception {
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ this.logFeederProps = logFeederProps;
LOG.info("init() called");
- checkPointExtension = LogFeederPropertiesUtil.getCheckPointExtension();
+ checkPointExtension = logFeederProps.getCheckPointExtension();
// Let's close the file and set it to true after we start monitoring it
setClosed(true);
@@ -86,7 +89,7 @@ public abstract class AbstractInputFile extends Input {
LOG.info("File to monitor " + logPath + ", tail=" + tail + ", isReady=" + isFileReady);
- super.init();
+ super.init(logFeederProps);
}
protected void processFile(File logPathFile, boolean follow) throws FileNotFoundException, IOException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
index 972011d..7b9dcd4 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogEntryCacheConfig;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.cache.LRUCache;
import org.apache.ambari.logfeeder.common.ConfigItem;
import org.apache.ambari.logfeeder.common.LogFeederException;
@@ -31,7 +33,6 @@ import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.output.Output;
import org.apache.ambari.logfeeder.output.OutputManager;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logsearch.config.api.model.inputconfig.Conditions;
import org.apache.ambari.logsearch.config.api.model.inputconfig.Fields;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
@@ -49,7 +50,7 @@ public abstract class Input extends ConfigItem implements Runnable {
protected InputManager inputManager;
protected OutputManager outputManager;
- private List<Output> outputList = new ArrayList<Output>();
+ private List<Output> outputList = new ArrayList<>();
private Thread thread;
private String type;
@@ -128,15 +129,15 @@ public abstract class Input extends ConfigItem implements Runnable {
}
@Override
- public void init() throws Exception {
- super.init();
- initCache();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
+ initCache(logFeederProps.getLogEntryCacheConfig());
tail = BooleanUtils.toBooleanDefaultIfNull(inputDescriptor.isTail(), DEFAULT_TAIL);
useEventMD5 = BooleanUtils.toBooleanDefaultIfNull(inputDescriptor.isUseEventMd5AsId(), DEFAULT_USE_EVENT_MD5);
genEventMD5 = BooleanUtils.toBooleanDefaultIfNull(inputDescriptor.isGenEventMd5(), DEFAULT_GEN_EVENT_MD5);
if (firstFilter != null) {
- firstFilter.init();
+ firstFilter.init(logFeederProps);
}
}
@@ -239,28 +240,28 @@ public abstract class Input extends ConfigItem implements Runnable {
}
}
- private void initCache() {
+ private void initCache(LogEntryCacheConfig cacheConfig) {
boolean cacheEnabled = inputDescriptor.isCacheEnabled() != null
? inputDescriptor.isCacheEnabled()
- : LogFeederPropertiesUtil.isCacheEnabled();
+ : cacheConfig.isCacheEnabled();
if (cacheEnabled) {
String cacheKeyField = inputDescriptor.getCacheKeyField() != null
? inputDescriptor.getCacheKeyField()
- : LogFeederPropertiesUtil.getCacheKeyField();
+ : cacheConfig.getCacheKeyField();
setCacheKeyField(cacheKeyField);
int cacheSize = inputDescriptor.getCacheSize() != null
? inputDescriptor.getCacheSize()
- : LogFeederPropertiesUtil.getCacheSize();
+ : cacheConfig.getCacheSize();
boolean cacheLastDedupEnabled = inputDescriptor.getCacheLastDedupEnabled() != null
? inputDescriptor.getCacheLastDedupEnabled()
- : LogFeederPropertiesUtil.isCacheLastDedupEnabled();
+ : cacheConfig.isCacheLastDedupEnabled();
long cacheDedupInterval = inputDescriptor.getCacheDedupInterval() != null
? inputDescriptor.getCacheDedupInterval()
- : Long.parseLong(LogFeederPropertiesUtil.getCacheDedupInterval());
+ : Long.parseLong(cacheConfig.getCacheDedupInterval());
setCache(new LRUCache(cacheSize, filePath, cacheDedupInterval, cacheLastDedupEnabled));
}
[18/51] [abbrv] ambari git commit: AMBARI-22647. ADDENDUM -Rafactor:
Package Log Search and Log Feeder into jars + cleanup start scripts
(oleewere)
Posted by am...@apache.org.
AMBARI-22647. ADDENDUM -Rafactor: Package Log Search and Log Feeder into jars + cleanup start scripts (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/afe469c9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/afe469c9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/afe469c9
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: afe469c98564aa7face62fe185eeb50b9b322e5f
Parents: 5c95c4a
Author: Oliver Szabo <ol...@gmail.com>
Authored: Mon Dec 18 22:01:09 2017 +0100
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Mon Dec 18 22:01:44 2017 +0100
----------------------------------------------------------------------
.../src/main/resources/logfeeder.properties | 5 +----
.../src/main/scripts/logfeeder.sh | 17 ++++++++---------
.../src/main/scripts/logsearch.sh | 17 ++++++++---------
ambari-logsearch/docker/bin/start.sh | 9 +++++----
4 files changed, 22 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/afe469c9/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 4b49446..5217309 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -23,9 +23,8 @@ logfeeder.config.files=${LOGFEEDER_RELATIVE_LOCATION:}target/classes/log-samples
${LOGFEEDER_RELATIVE_LOCATION:}target/classes/log-samples/shipper-conf/output.config-sample.json
logfeeder.log.filter.enable=true
+logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
-logfeeder.solr.config.interval=5
-logfeeder.solr.core.config.name=history
logfeeder.solr.zk_connect_string=localhost:2181
logfeeder.cache.enabled=true
@@ -34,7 +33,5 @@ logfeeder.cache.key.field=log_message
logfeeder.cache.dedup.interval=1000
logfeeder.cache.last.dedup.enabled=true
-logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
-
#logfeeder tmp dir
logfeeder.tmp.dir=${LOGFEEDER_RELATIVE_LOCATION:}target/tmp
http://git-wip-us.apache.org/repos/asf/ambari/blob/afe469c9/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
index cfacea5..5f4edea 100755
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
@@ -119,27 +119,26 @@ function spinner() {
}
function status() {
- echo "Checking Log Feeder status ..."
+ echo "Checking Log Feeder status ..." >&2
if [ -f "$LOGFEEDER_PID_FILE" ]; then
LOGFEEDER_PID=`cat "$LOGFEEDER_PID_FILE"`
else
- echo "Log Feeder pid not exists. (probably the process is not running)"
- exit 1
+ echo "Log Feeder pid not exists. (probably the process is not running)" >&2
+ return 1
fi
if ps -p $LOGFEEDER_PID > /dev/null
then
- echo "Log Feeder process is running. (pid: $LOGFEEDER_PID)"
- exit 0
+ echo "Log Feeder process is running. (pid: $LOGFEEDER_PID)" >&2
+ return 0
else
- echo "Log Feeder process is not running."
- exit 1
+ echo "Log Feeder process is not running." >&2
+ return 1
fi
}
function start() {
- (status);
- exit_status=$(echo $?)
+ exit_status=$(status; echo $?)
if [ "$exit_status" = "0" ]; then
echo "Skipping start process."
exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/afe469c9/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
index 128617c..8439065 100755
--- a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch.sh
@@ -109,27 +109,26 @@ function spinner() {
}
function status() {
- echo "Checking Log Search status ..."
+ echo "Checking Log Search status ..." >&2
if [ -f "$LOGSEARCH_PID_FILE" ]; then
LOGSEARCH_PID=`cat "$LOGSEARCH_PID_FILE"`
else
- echo "Log Search pid not exists. (probably the process is not running)"
- exit 1
+ echo "Log Search pid not exists. (probably the process is not running)" >&2
+ return 1
fi
if ps -p $LOGSEARCH_PID > /dev/null
then
- echo "Log Search process is running. (pid: $LOGSEARCH_PID)"
- exit 0
+ echo "Log Search process is running. (pid: $LOGSEARCH_PID)" >&2
+ return 0
else
- echo "Log Search process is not running."
- exit 1
+ echo "Log Search process is not running." >&2
+ return 1
fi
}
function start() {
- (status);
- exit_status=$(echo $?)
+ exit_status=$(status; echo $?)
if [ "$exit_status" = "0" ]; then
echo "Skipping start process."
exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/afe469c9/ambari-logsearch/docker/bin/start.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh
index f2acf9b..6ba20fb 100644
--- a/ambari-logsearch/docker/bin/start.sh
+++ b/ambari-logsearch/docker/bin/start.sh
@@ -46,7 +46,7 @@ function create_logfeeder_configs() {
}
function create_logsearch_configs() {
- mkdir /root/config/logsearch
+ mkdir -p /root/config/logsearch
cp /root/test-config/logsearch/log4j.xml /root/config/logsearch/
cp /root/test-config/logsearch/logsearch-env.sh /root/config/logsearch/
cp $LOGSEARCH_SERVER_PATH/conf/user_pass.json /root/config/logsearch/user_pass.json
@@ -148,7 +148,7 @@ function main() {
case $component in
"solr")
create_solr_configs
- echo "Start Solr only.."
+ echo "Start Solr only ..."
export COMPONENT_LOG="solr"
generate_keys
start_solr_d
@@ -156,17 +156,18 @@ function main() {
;;
"logfeeder")
create_logfeeder_configs
- echo "Start Log Feeder only.."
+ echo "Start Log Feeder only ..."
export COMPONENT_LOG="logfeeder"
generate_keys
start_logfeeder
;;
"logsearch")
create_logsearch_configs
- echo "Start Log Search only.."
+ echo "Start Log Search only ..."
export COMPONENT_LOG="logsearch"
generate_keys
start_logsearch
+ log
;;
*)
create_configs
[08/51] [abbrv] ambari git commit: AMBARI-22659. unable to proceed
with cluster install after component install fails. (mpapirkovskyy)
Posted by am...@apache.org.
AMBARI-22659. unable to proceed with cluster install after component install fails. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/856d9a53
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/856d9a53
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/856d9a53
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 856d9a53d39b1658a846311d672ba54699b73f90
Parents: 3ab1045
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Fri Dec 15 18:13:49 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Fri Dec 15 18:46:35 2017 +0200
----------------------------------------------------------------------
.../apache/ambari/server/orm/dao/HostVersionDAO.java | 15 +++++++++++++++
.../server/orm/entities/HostVersionEntity.java | 5 +++++
.../ambari/server/state/cluster/ClusterImpl.java | 10 ++++++++++
.../ambari/server/orm/dao/HostVersionDAOTest.java | 12 ++++++++++++
4 files changed, 42 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/856d9a53/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
index 158370f..feb4172 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java
@@ -130,6 +130,21 @@ public class HostVersionDAO extends CrudDAO<HostVersionEntity, Long> {
}
/**
+ * Retrieve all of the cluster versions for the given cluster name.
+ *
+ * @param clusterName Cluster name
+ * @return Return all of the host versions that match the criteria.
+ */
+ @RequiresSession
+ public List<HostVersionEntity> findByCluster(String clusterName) {
+ final TypedQuery<HostVersionEntity> query = entityManagerProvider.get()
+ .createNamedQuery("findByCluster", HostVersionEntity.class);
+ query.setParameter("clusterName", clusterName);
+
+ return daoUtils.selectList(query);
+ }
+
+ /**
* Retrieve all of the host versions for the given cluster name, and state. <br/>
* @param clusterName Cluster name
* @param state repository version state
http://git-wip-us.apache.org/repos/asf/ambari/blob/856d9a53/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
index 4a030af..2c61a3c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java
@@ -66,6 +66,11 @@ import org.apache.ambari.server.state.RepositoryVersionState;
query = "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity host JOIN host.clusterEntities clusters "
+ "WHERE clusters.clusterName=:clusterName AND hostVersion.state=:state"),
+ @NamedQuery(
+ name = "findByCluster",
+ query = "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity host JOIN host.clusterEntities clusters "
+ + "WHERE clusters.clusterName=:clusterName"),
+
@NamedQuery(name = "hostVersionByClusterHostnameAndState", query =
"SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity host JOIN host.clusterEntities clusters " +
"WHERE clusters.clusterName=:clusterName AND hostVersion.hostEntity.hostName=:hostName AND hostVersion.state=:state"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/856d9a53/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index c23b971..2266c62 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -1334,6 +1334,7 @@ public class ClusterImpl implements Cluster {
try {
refresh();
deleteAllServices();
+ resetHostVersions();
refresh(); // update one-to-many clusterServiceEntities
removeEntities();
@@ -1353,6 +1354,15 @@ public class ClusterImpl implements Cluster {
clusterDAO.removeByPK(clusterId);
}
+ private void resetHostVersions() {
+ for (HostVersionEntity hostVersionEntity : hostVersionDAO.findByCluster(getClusterName())) {
+ if (!hostVersionEntity.getState().equals(RepositoryVersionState.NOT_REQUIRED)) {
+ hostVersionEntity.setState(RepositoryVersionState.NOT_REQUIRED);
+ hostVersionDAO.merge(hostVersionEntity);
+ }
+ }
+ }
+
@Override
public ServiceConfigVersionResponse addDesiredConfig(String user, Set<Config> configs) {
return addDesiredConfig(user, configs, null);
http://git-wip-us.apache.org/repos/asf/ambari/blob/856d9a53/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostVersionDAOTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostVersionDAOTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostVersionDAOTest.java
index bb077d6..88f42d3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostVersionDAOTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostVersionDAOTest.java
@@ -247,6 +247,18 @@ public class HostVersionDAOTest {
}
/**
+ * Test the {@link HostVersionDAO#findByCluster(String)} method.
+ */
+ @Test
+ public void testFindByCluster() {
+ Assert.assertEquals(3, hostVersionDAO.findByCluster("test_cluster1").size());
+
+ addMoreVersions();
+
+ Assert.assertEquals(9, hostVersionDAO.findByCluster("test_cluster1").size());
+ }
+
+ /**
* Test the {@link HostVersionDAO#findByClusterHostAndState(String, String, org.apache.ambari.server.state.RepositoryVersionState)} method.
*/
@Test
[34/51] [abbrv] ambari git commit: AMBARI-22306. Set
VersionAdvertised as false for superset in tech-preview (nishantmonu51)
Posted by am...@apache.org.
AMBARI-22306. Set VersionAdvertised as false for superset in tech-preview (nishantmonu51)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a37985b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a37985b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a37985b
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 6a37985bcdab81a9c5f48b7275b951662856fa91
Parents: 26bcc97
Author: Nishant <ni...@gmail.com>
Authored: Fri Dec 22 00:39:52 2017 +0530
Committer: Nishant <ni...@gmail.com>
Committed: Fri Dec 22 00:39:52 2017 +0530
----------------------------------------------------------------------
.../common-services/SUPERSET/0.15.0/metainfo.xml | 2 +-
.../stacks/HDP/2.0.6/properties/stack_packages.json | 14 ++++++++++++++
2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a37985b/ambari-server/src/main/resources/common-services/SUPERSET/0.15.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SUPERSET/0.15.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SUPERSET/0.15.0/metainfo.xml
index 5c6ed11..849bfbe 100644
--- a/ambari-server/src/main/resources/common-services/SUPERSET/0.15.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SUPERSET/0.15.0/metainfo.xml
@@ -29,7 +29,7 @@
<displayName>Superset</displayName>
<category>MASTER</category>
<cardinality>1+</cardinality>
- <versionAdvertised>true</versionAdvertised>
+ <versionAdvertised>false</versionAdvertised>
<commandScript>
<script>scripts/superset.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a37985b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
index 6a0eadd..dc71b4d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
@@ -864,6 +864,20 @@
]
}
},
+ "SUPERSET": {
+ "SUPERSET": {
+ "STACK-SELECT-PACKAGE": "superset",
+ "INSTALL": [
+ "superset"
+ ],
+ "PATCH": [
+ "superset"
+ ],
+ "STANDARD": [
+ "superset"
+ ]
+ }
+ },
"TITAN": {
"TITAN_CLIENT": {
"STACK-SELECT-PACKAGE": "titan-client",
[29/51] [abbrv] ambari git commit: AMBARI-22530. Refactor internal
code of handling info between kerberos wizard actions (echekanskiy)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
index 5ec4c10..a803dcf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
@@ -34,10 +34,11 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
-import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
import org.apache.ambari.server.serveraction.ActionLog;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.directory.server.kerberos.shared.keytab.Keytab;
import org.slf4j.Logger;
@@ -52,7 +53,7 @@ import com.google.inject.Inject;
* This class mainly relies on the KerberosServerAction to iterate through metadata identifying
* the Kerberos keytab files that need to be created. For each identity in the metadata, this
* implementation's
- * {@link KerberosServerAction#processIdentity(Map, String, KerberosOperationHandler, Map, Map)}
+ * {@link KerberosServerAction#processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, Map)}
* is invoked attempting the creation of the relevant keytab file.
*/
public class CreateKeytabFilesServerAction extends KerberosServerAction {
@@ -65,12 +66,6 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
private KerberosPrincipalDAO kerberosPrincipalDAO;
/**
- * KerberosPrincipalHostDAO used to get Kerberos principal details
- */
- @Inject
- private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
-
- /**
* Configuration used to get the configured properties such as the keytab file cache directory
*/
@Inject
@@ -82,6 +77,9 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
@Inject
private HostDAO hostDAO;
+ @Inject
+ private KerberosKeytabController kerberosKeytabController;
+
/**
* A map of data used to track what has been processed in order to optimize the creation of keytabs
* such as knowing when to create a cached keytab file or use a cached keytab file.
@@ -118,10 +116,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
* If a password exists for the current evaluatedPrincipal, use a
* {@link org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler} to generate
* the keytab file. To help avoid filename collisions and to build a structure that is easy to
- * discover, each keytab file is stored in host-specific
- * ({@link org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileReader#HOSTNAME})
- * directory using the SHA1 hash of its destination file path
- * ({@link org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileReader#KEYTAB_FILE_PATH})
+ * discover, each keytab file is stored in host-specific directory using the SHA1 hash of its destination file path.
* <p/>
* <pre>
* data_directory
@@ -133,8 +128,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
* | |- ...
* </pre>
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -145,7 +139,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
* @throws AmbariException if an error occurs while processing the identity record
*/
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
@@ -160,40 +154,42 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
CommandReport commandReport = null;
String message = null;
- try {
- if (identityRecord != null) {
- String dataDirectory = getDataDirectoryPath();
- if (operationHandler == null) {
- message = String.format("Failed to create keytab file for %s, missing KerberosOperationHandler", evaluatedPrincipal);
- actionLog.writeStdErr(message);
- LOG.error(message);
- commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
- } else if (dataDirectory == null) {
- message = "The data directory has not been set. Generated keytab files can not be stored.";
- LOG.error(message);
- commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
- } else {
- Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
- Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
+ Set<ResolvedKerberosKeytab> keytabsToCreate = kerberosKeytabController.getFromPrincipal(resolvedPrincipal);
- String hostName = identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME);
- String keytabFilePath = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
+ try {
+ String dataDirectory = getDataDirectoryPath();
+
+ if (operationHandler == null) {
+ message = String.format("Failed to create keytab file for %s, missing KerberosOperationHandler", resolvedPrincipal.getPrincipal());
+ actionLog.writeStdErr(message);
+ LOG.error(message);
+ commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+ } else if (dataDirectory == null) {
+ message = "The data directory has not been set. Generated keytab files can not be stored.";
+ LOG.error(message);
+ commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+ } else {
+ Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
+ Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
+ for (ResolvedKerberosKeytab rkk : keytabsToCreate) {
+ String hostName = resolvedPrincipal.getHostName();
+ String keytabFilePath = rkk.getFile();
if ((hostName != null) && !hostName.isEmpty() && (keytabFilePath != null) && !keytabFilePath.isEmpty()) {
- Set<String> visitedPrincipalKeys = visitedIdentities.get(evaluatedPrincipal);
+ Set<String> visitedPrincipalKeys = visitedIdentities.get(resolvedPrincipal.getPrincipal());
String visitationKey = String.format("%s|%s", hostName, keytabFilePath);
if ((visitedPrincipalKeys == null) || !visitedPrincipalKeys.contains(visitationKey)) {
// Look up the current evaluatedPrincipal's password.
// If found create the keytab file, else try to find it in the cache.
- String password = principalPasswordMap.get(evaluatedPrincipal);
- Integer keyNumber = principalKeyNumberMap.get(evaluatedPrincipal);
+ String password = principalPasswordMap.get(resolvedPrincipal.getPrincipal());
+ Integer keyNumber = principalKeyNumberMap.get(resolvedPrincipal.getPrincipal());
- message = String.format("Creating keytab file for %s on host %s", evaluatedPrincipal, hostName);
+ message = String.format("Creating keytab file for %s on host %s", resolvedPrincipal.getPrincipal(), hostName);
LOG.info(message);
actionLog.writeStdOut(message);
- auditEventBuilder.withPrincipal(evaluatedPrincipal).withHostName(hostName).withKeyTabFilePath(keytabFilePath);
+ auditEventBuilder.withPrincipal(resolvedPrincipal.getPrincipal()).withHostName(hostName).withKeyTabFilePath(keytabFilePath);
// Determine where to store the keytab file. It should go into a host-specific
// directory under the previously determined data directory.
@@ -206,32 +202,22 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
}
if (hostDirectory.exists()) {
- File destinationKeytabFile = new File(hostDirectory, DigestUtils.sha1Hex(keytabFilePath));
- HostEntity hostEntity = hostDAO.findByName(hostName);
- // in case of ambari-server identity there's no host entity for ambari_server host
- if (hostEntity == null && !hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
- message = "Failed to find HostEntity for hostname = " + hostName;
- actionLog.writeStdErr(message);
- LOG.error(message);
- commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
- return commandReport;
- }
+ File destinationKeytabFile = new File(hostDirectory, DigestUtils.sha256Hex(keytabFilePath));
boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL;
- KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+ KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(resolvedPrincipal.getPrincipal());
String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath();
if (password == null) {
- if (!regenerateKeytabs && (hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || kerberosPrincipalHostDAO
- .exists(evaluatedPrincipal, hostEntity.getHostId(), keytabFilePath)) && cachedKeytabPath == null) {
+ if (!regenerateKeytabs && hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
// There is nothing to do for this since it must already exist and we don't want to
// regenerate the keytab
- message = String.format("Skipping keytab file for %s, missing password indicates nothing to do", evaluatedPrincipal);
+ message = String.format("Skipping keytab file for %s, missing password indicates nothing to do", resolvedPrincipal.getPrincipal());
LOG.debug(message);
} else {
if (cachedKeytabPath == null) {
- message = String.format("Failed to create keytab for %s, missing cached file", evaluatedPrincipal);
+ message = String.format("Failed to create keytab for %s, missing cached file", resolvedPrincipal.getPrincipal());
actionLog.writeStdErr(message);
LOG.error(message);
commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
@@ -239,7 +225,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
try {
operationHandler.createKeytabFile(new File(cachedKeytabPath), destinationKeytabFile);
} catch (KerberosOperationException e) {
- message = String.format("Failed to create keytab file for %s - %s", evaluatedPrincipal, e.getMessage());
+ message = String.format("Failed to create keytab file for %s - %s", resolvedPrincipal.getPrincipal(), e.getMessage());
actionLog.writeStdErr(message);
LOG.error(message, e);
commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
@@ -247,24 +233,24 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
}
}
} else {
- Keytab keytab = createKeytab(evaluatedPrincipal, password, keyNumber, operationHandler, visitedPrincipalKeys != null, true, actionLog);
+ Keytab keytab = createKeytab(resolvedPrincipal.getPrincipal(), password, keyNumber, operationHandler, visitedPrincipalKeys != null, true, actionLog);
if (keytab != null) {
try {
if (operationHandler.createKeytabFile(keytab, destinationKeytabFile)) {
ensureAmbariOnlyAccess(destinationKeytabFile);
- message = String.format("Successfully created keytab file for %s at %s", evaluatedPrincipal, destinationKeytabFile.getAbsolutePath());
+ message = String.format("Successfully created keytab file for %s at %s", resolvedPrincipal.getPrincipal(), destinationKeytabFile.getAbsolutePath());
LOG.debug(message);
- auditEventBuilder.withPrincipal(evaluatedPrincipal).withHostName(hostName).withKeyTabFilePath(destinationKeytabFile.getAbsolutePath());
+ auditEventBuilder.withPrincipal(resolvedPrincipal.getPrincipal()).withHostName(hostName).withKeyTabFilePath(destinationKeytabFile.getAbsolutePath());
} else {
- message = String.format("Failed to create keytab file for %s at %s", evaluatedPrincipal, destinationKeytabFile.getAbsolutePath());
+ message = String.format("Failed to create keytab file for %s at %s", resolvedPrincipal.getPrincipal(), destinationKeytabFile.getAbsolutePath());
actionLog.writeStdErr(message);
LOG.error(message);
commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
}
} catch (KerberosOperationException e) {
- message = String.format("Failed to create keytab file for %s - %s", evaluatedPrincipal, e.getMessage());
+ message = String.format("Failed to create keytab file for %s - %s", resolvedPrincipal.getPrincipal(), e.getMessage());
actionLog.writeStdErr(message);
LOG.error(message, e);
commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
@@ -275,20 +261,20 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
if (visitedPrincipalKeys == null) {
visitedPrincipalKeys = new HashSet<>();
- visitedIdentities.put(evaluatedPrincipal, visitedPrincipalKeys);
+ visitedIdentities.put(resolvedPrincipal.getPrincipal(), visitedPrincipalKeys);
}
visitedPrincipalKeys.add(visitationKey);
}
} else {
message = String.format("Failed to create keytab file for %s, the container directory does not exist: %s",
- evaluatedPrincipal, hostDirectory.getAbsolutePath());
+ resolvedPrincipal.getPrincipal(), hostDirectory.getAbsolutePath());
actionLog.writeStdErr(message);
LOG.error(message);
commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
}
} else {
- LOG.debug("Skipping previously processed keytab for {} on host {}", evaluatedPrincipal, hostName);
+ LOG.debug("Skipping previously processed keytab for {} on host {}", resolvedPrincipal.getPrincipal(), hostName);
}
}
}
@@ -420,7 +406,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
}
}
- File cachedKeytabFile = new File(cacheDirectory, DigestUtils.sha1Hex(principal + String.valueOf(System.currentTimeMillis())));
+ File cachedKeytabFile = new File(cacheDirectory, DigestUtils.sha256Hex(principal + String.valueOf(System.currentTimeMillis())));
try {
keytab.write(cachedKeytabFile);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
index 0c90659..a108c9b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
@@ -28,12 +28,13 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.audit.event.kerberos.CreatePrincipalKerberosAuditEvent;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
-import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
import org.apache.ambari.server.security.SecurePasswordHelper;
import org.apache.ambari.server.serveraction.ActionLog;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +46,7 @@ import com.google.inject.Inject;
* <p/>
* This class mainly relies on the KerberosServerAction to iterate through metadata identifying
* the Kerberos principals that need to be created. For each identity in the metadata, this implementation's
- * {@link KerberosServerAction#processIdentity(Map, String, KerberosOperationHandler, Map, Map)}
+ * {@link KerberosServerAction#processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, Map)}
* is invoked attempting the creation of the relevant principal.
*/
public class CreatePrincipalsServerAction extends KerberosServerAction {
@@ -58,17 +59,14 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
private KerberosPrincipalDAO kerberosPrincipalDAO;
/**
- * KerberosPrincipalHostDAO used to get Kerberos principal details
- */
- @Inject
- private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
-
- /**
* SecurePasswordHelper used to generate secure passwords for newly created principals
*/
@Inject
private SecurePasswordHelper securePasswordHelper;
+ @Inject
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
/**
* A set of visited principal names used to prevent unnecessary processing on already processed
* principal names
@@ -106,8 +104,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
* store the new key numbers in the shared principal-to-key_number map so that subsequent process
* may use the data if necessary.
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -118,7 +115,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
* @throws AmbariException if an error occurs while processing the identity record
*/
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
@@ -126,16 +123,16 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
CommandReport commandReport = null;
// Only process this principal name if we haven't already processed it
- if (!seenPrincipals.contains(evaluatedPrincipal)) {
- seenPrincipals.add(evaluatedPrincipal);
+ // TODO optimize - split invalidation and principal creation to separate stages
+ if (!seenPrincipals.contains(resolvedPrincipal.getPrincipal())) {
+ seenPrincipals.add(resolvedPrincipal.getPrincipal());
boolean processPrincipal;
- // TODO add invalidate_principals option to make keytabs invalid all over the cluster.
- KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+ KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(resolvedPrincipal.getPrincipal());
boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL;
- boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
+ boolean servicePrincipal = resolvedPrincipal.isService();
if (regenerateKeytabs) {
// force recreation of principal due to keytab regeneration
// regenerate only service principals if request filtered by hosts
@@ -154,24 +151,24 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
if (processPrincipal) {
Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
- String password = principalPasswordMap.get(evaluatedPrincipal);
+ String password = principalPasswordMap.get(resolvedPrincipal.getPrincipal());
if (password == null) {
- CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, regenerateKeytabs, actionLog);
+ CreatePrincipalResult result = createPrincipal(resolvedPrincipal.getPrincipal(), servicePrincipal, kerberosConfiguration, operationHandler, regenerateKeytabs, actionLog);
if (result == null) {
commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
} else {
Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
- principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
- principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber());
+ principalPasswordMap.put(resolvedPrincipal.getPrincipal(), result.getPassword());
+ principalKeyNumberMap.put(resolvedPrincipal.getPrincipal(), result.getKeyNumber());
// invalidate given principal for all keytabs to make them redistributed again
- for (KerberosPrincipalHostEntity kphe: kerberosPrincipalHostDAO.findByPrincipal(evaluatedPrincipal)) {
- kphe.setDistributed(false);
- kerberosPrincipalHostDAO.merge(kphe);
+ for (KerberosKeytabPrincipalEntity kkpe: kerberosKeytabPrincipalDAO.findByPrincipal(resolvedPrincipal.getPrincipal())) {
+ kkpe.setDistributed(false);
+ kerberosKeytabPrincipalDAO.merge(kkpe);
}
// invalidate principal cache
- KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+ KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(resolvedPrincipal.getPrincipal());
try {
new File(principalEntity.getCachedKeytabPath()).delete();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
index 4c80bd4..7c28494 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
@@ -29,8 +29,13 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.audit.event.kerberos.DestroyPrincipalKerberosAuditEvent;
import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
+import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.utils.ShellCommandUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +48,7 @@ import com.google.inject.Inject;
* This class mainly relies on the KerberosServerAction to iterate through metadata identifying
* the Kerberos principals that need to be removed from the relevant KDC. For each identity in the
* metadata, this implementation's
- * {@link KerberosServerAction#processIdentity(Map, String, KerberosOperationHandler, Map, Map)}
+ * {@link KerberosServerAction#processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, Map)}
* is invoked attempting the removal of the relevant principal.
*/
public class DestroyPrincipalsServerAction extends KerberosServerAction {
@@ -52,6 +57,12 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
@Inject
private KerberosPrincipalDAO kerberosPrincipalDAO;
+ @Inject
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
+ @Inject
+ private KerberosKeytabDAO kerberosKeytabDAO;
+
/**
* A set of visited principal names used to prevent unnecessary processing on already processed
* principal names
@@ -81,8 +92,7 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
/**
* For each identity, remove the principal from the configured KDC.
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -93,69 +103,73 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
* @throws org.apache.ambari.server.AmbariException if an error occurs while processing the identity record
*/
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
throws AmbariException {
// Only process this principal if we haven't already processed it
- if (!seenPrincipals.contains(evaluatedPrincipal)) {
- seenPrincipals.add(evaluatedPrincipal);
+ if (!seenPrincipals.contains(resolvedPrincipal.getPrincipal())) {
+ seenPrincipals.add(resolvedPrincipal.getPrincipal());
- String message = String.format("Destroying identity, %s", evaluatedPrincipal);
+ String message = String.format("Destroying identity, %s", resolvedPrincipal.getPrincipal());
LOG.info(message);
actionLog.writeStdOut(message);
DestroyPrincipalKerberosAuditEvent.DestroyPrincipalKerberosAuditEventBuilder auditEventBuilder = DestroyPrincipalKerberosAuditEvent.builder()
.withTimestamp(System.currentTimeMillis())
.withRequestId(getHostRoleCommand().getRequestId())
.withTaskId(getHostRoleCommand().getTaskId())
- .withPrincipal(evaluatedPrincipal);
+ .withPrincipal(resolvedPrincipal.getPrincipal());
try {
try {
- boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
- operationHandler.removePrincipal(evaluatedPrincipal, servicePrincipal);
+ boolean servicePrincipal = resolvedPrincipal.isService();
+ operationHandler.removePrincipal(resolvedPrincipal.getPrincipal(), servicePrincipal);
} catch (KerberosOperationException e) {
- message = String.format("Failed to remove identity for %s from the KDC - %s", evaluatedPrincipal, e.getMessage());
+ message = String.format("Failed to remove identity for %s from the KDC - %s", resolvedPrincipal.getPrincipal(), e.getMessage());
LOG.warn(message);
actionLog.writeStdErr(message);
auditEventBuilder.withReasonOfFailure(message);
}
try {
- KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+ KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(resolvedPrincipal.getPrincipal());
if (principalEntity != null) {
String cachedKeytabPath = principalEntity.getCachedKeytabPath();
-
+ KerberosKeytabEntity kke = kerberosKeytabDAO.find(resolvedPrincipal.getResolvedKerberosKeytab().getFile());
+ kerberosKeytabDAO.remove(kke);
kerberosPrincipalDAO.remove(principalEntity);
// If a cached keytabs file exists for this principal, delete it.
if (cachedKeytabPath != null) {
if (!new File(cachedKeytabPath).delete()) {
- LOG.debug("Failed to remove cached keytab for {}", evaluatedPrincipal);
+ LOG.debug("Failed to remove cached keytab for {}", resolvedPrincipal.getPrincipal());
}
}
}
// delete Ambari server keytab
- String hostName = identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME);
+ String hostName = resolvedPrincipal.getHostName();
if (hostName != null && hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
- String keytabFilePath = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
- if (keytabFilePath != null) {
- try {
- ShellCommandUtil.Result result = ShellCommandUtil.delete(keytabFilePath, true, true);
- if (!result.isSuccessful()) {
- LOG.warn("Failed to remove ambari keytab for {} due to {}", evaluatedPrincipal, result.getStderr());
+ ResolvedKerberosKeytab resolvedKeytab = resolvedPrincipal.getResolvedKerberosKeytab();
+ if (resolvedKeytab != null) {
+ String keytabFilePath = resolvedKeytab.getFile();
+ if (keytabFilePath != null) {
+ try {
+ ShellCommandUtil.Result result = ShellCommandUtil.delete(keytabFilePath, true, true);
+ if (!result.isSuccessful()) {
+ LOG.warn("Failed to remove ambari keytab for {} due to {}", resolvedPrincipal.getPrincipal(), result.getStderr());
+ }
+ } catch (IOException|InterruptedException e) {
+ LOG.warn("Failed to remove ambari keytab for " + resolvedPrincipal.getPrincipal(), e);
}
- } catch (IOException|InterruptedException e) {
- LOG.warn("Failed to remove ambari keytab for " + evaluatedPrincipal, e);
}
}
}
} catch (Throwable t) {
- message = String.format("Failed to remove identity for %s from the Ambari database - %s", evaluatedPrincipal, t.getMessage());
+ message = String.format("Failed to remove identity for %s from the Ambari database - %s", resolvedPrincipal.getPrincipal(), t.getMessage());
LOG.warn(message);
actionLog.writeStdErr(message);
auditEventBuilder.withReasonOfFailure(message);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
index bfd5e40..225e53e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
@@ -26,9 +26,10 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.utils.ShellCommandUtil;
import org.apache.ambari.server.utils.StageUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,8 +44,7 @@ public class FinalizeKerberosServerAction extends KerberosServerAction {
* some user accounts and groups may not have been available (at the OS level) when the keytab files
* were created.
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -54,39 +54,39 @@ public class FinalizeKerberosServerAction extends KerberosServerAction {
* @throws AmbariException
*/
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
throws AmbariException {
- if (identityRecord != null) {
+ if (resolvedPrincipal != null) {
// If the record's HOSTNAME value is "ambari-server", rather than an actual hostname it will
// not match the Ambari server's host name. This will occur if the there is no agent installed
// on the Ambari server host. This is ok, since any keytab files installed on the Ambari server
// host will already have the permissions set so that only the Ambari server can read it.
// There is no need to update the permissions for those keytab files so that installed services
// can access them since no services will be installed on the host.
- if (StageUtils.getHostName().equals(identityRecord.get(KerberosIdentityDataFile.HOSTNAME))) {
+ if (StageUtils.getHostName().equals(resolvedPrincipal.getHostName())) {
// If the principal name exists in one of the shared data maps, it has been processed by the
// current "Enable Kerberos" or "Add component" workflow and therefore should already have
// the correct permissions assigned. The relevant keytab files can be skipped.
Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
- if ((principalPasswordMap == null) || !principalPasswordMap.containsKey(evaluatedPrincipal)) {
+ if ((principalPasswordMap == null) || !principalPasswordMap.containsKey(resolvedPrincipal.getPrincipal())) {
- String keytabFilePath = identityRecord.get(KerberosIdentityDataFile.KEYTAB_FILE_PATH);
+ String keytabFilePath = resolvedPrincipal.getKeytabPath();
if (!StringUtils.isEmpty(keytabFilePath)) {
Set<String> visited = (Set<String>) requestSharedDataContext.get(this.getClass().getName() + "_visited");
if (!visited.contains(keytabFilePath)) {
- String ownerName = identityRecord.get(KerberosIdentityDataFile.KEYTAB_FILE_OWNER_NAME);
- String ownerAccess = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS);
+ String ownerName = resolvedPrincipal.getResolvedKerberosKeytab().getOwnerName();
+ String ownerAccess = resolvedPrincipal.getResolvedKerberosKeytab().getOwnerAccess();
boolean ownerWritable = "w".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
boolean ownerReadable = "r".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
- String groupName = identityRecord.get(KerberosIdentityDataFile.KEYTAB_FILE_GROUP_NAME);
- String groupAccess = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_OWNER_ACCESS);
+ String groupName = resolvedPrincipal.getResolvedKerberosKeytab().getGroupName();
+ String groupAccess = resolvedPrincipal.getResolvedKerberosKeytab().getGroupAccess();
boolean groupWritable = "w".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
boolean groupReadable = "r".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
index ff5f5ce..2c9aa8c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
@@ -18,11 +18,10 @@
package org.apache.ambari.server.serveraction.kerberos;
-import static org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileReader.DATA_FILE_NAME;
-
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -36,6 +35,9 @@ import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.utils.StageUtils;
@@ -178,6 +180,10 @@ public abstract class KerberosServerAction extends AbstractServerAction {
@Inject
HostDAO hostDAO;
+
+ @Inject
+ KerberosKeytabController kerberosKeytabController;
+
/**
* Given a (command parameter) Map and a property name, attempts to safely retrieve the requested
* data.
@@ -235,10 +241,9 @@ public abstract class KerberosServerAction extends AbstractServerAction {
*/
protected static OperationType getOperationType(Map<String, String> commandParameters) {
String value = getCommandParameterValue(commandParameters, OPERATION_TYPE);
- if(StringUtils.isEmpty(value)) {
+ if (StringUtils.isEmpty(value)) {
return OperationType.DEFAULT;
- }
- else {
+ } else {
return OperationType.valueOf(value.toUpperCase());
}
}
@@ -365,14 +370,32 @@ public abstract class KerberosServerAction extends AbstractServerAction {
}
/**
+ * Returns preconfigure type passed to current action.
+ *
+ * @return PreconfigureServiceType
+ */
+ protected PreconfigureServiceType getCommandPreconfigureType() {
+ String preconfigureServices = getCommandParameterValue(getCommandParameters(), PRECONFIGURE_SERVICES);
+ PreconfigureServiceType type = null;
+ if (!StringUtils.isEmpty(preconfigureServices)) {
+ try {
+ type = PreconfigureServiceType.valueOf(preconfigureServices.toUpperCase());
+ } catch (Throwable t) {
+ LOG.warn("Invalid preconfigure_services value, assuming DEFAULT: {}", preconfigureServices);
+ type = PreconfigureServiceType.DEFAULT;
+ }
+ }
+ return type;
+ }
+
+ /**
* Iterates through the Kerberos identity metadata from the
* {@link org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileReader} and calls
* the implementing class to handle each identity found.
* <p/>
- * Using the "data_directory" value from this action's command parameters map, creates a
- * {@link KerberosIdentityDataFileReader} to parse
- * the relative identity.dat file and iterate through its "records". Each "record" is process using
- * {@link #processRecord(Map, String, KerberosOperationHandler, Map, Map)}.
+ * Using {@link #getHostFilter()}, {@link #getIdentityFilter()} and {@link #getServiceComponentFilter()} it retrieve
+ * list of filtered keytabs and their principals and process each principal using
+ * {@link #processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, Map)}.
*
* @param requestSharedDataContext a Map to be used a shared data among all ServerActions related
* to a given request
@@ -390,102 +413,43 @@ public abstract class KerberosServerAction extends AbstractServerAction {
if (commandParameters != null) {
// Grab the relevant data from this action's command parameters map
PrincipalKeyCredential administratorCredential = kerberosHelper.getKDCAdministratorCredentials(getClusterName());
- String defaultRealm = getDefaultRealm(commandParameters);
KDCType kdcType = getKDCType(commandParameters);
- String dataDirectoryPath = getDataDirectoryPath(commandParameters);
-
- if (dataDirectoryPath != null) {
- File dataDirectory = new File(dataDirectoryPath);
-
- // If the data directory exists, attempt to process further, else assume there is no work to do
- if (dataDirectory.exists()) {
- if (!dataDirectory.isDirectory() || !dataDirectory.canRead()) {
- String message = String.format("Failed to process the identities, the data directory is not accessible: %s",
- dataDirectory.getAbsolutePath());
- actionLog.writeStdErr(message);
- LOG.error(message);
- throw new AmbariException(message);
- }
- // The "identity data" file may or may not exist in the data directory, depending on if
- // there is work to do or not.
- File identityDataFile = new File(dataDirectory, DATA_FILE_NAME);
-
- if (identityDataFile.exists()) {
- if (!identityDataFile.canRead()) {
- String message = String.format("Failed to process the identities, cannot read the index file: %s",
- identityDataFile.getAbsolutePath());
- actionLog.writeStdErr(message);
- LOG.error(message);
- throw new AmbariException(message);
- }
-
- KerberosOperationHandler handler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kdcType);
- if (handler == null) {
- String message = String.format("Failed to process the identities, a KDC operation handler was not found for the KDC type of : %s",
- kdcType.toString());
- actionLog.writeStdErr(message);
- LOG.error(message);
- throw new AmbariException(message);
- }
-
- Map<String, String> kerberosConfiguration = getConfiguration("kerberos-env");
+ String defaultRealm = getDefaultRealm(commandParameters);
- try {
- handler.open(administratorCredential, defaultRealm, kerberosConfiguration);
- } catch (KerberosOperationException e) {
- String message = String.format("Failed to process the identities, could not properly open the KDC operation handler: %s",
- e.getMessage());
- actionLog.writeStdErr(message);
- LOG.error(message);
- throw new AmbariException(message, e);
- }
+ KerberosOperationHandler handler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kdcType);
+ Map<String, String> kerberosConfiguration = getConfiguration("kerberos-env");
+
+ try {
+ handler.open(administratorCredential, defaultRealm, kerberosConfiguration);
+ } catch (KerberosOperationException e) {
+ String message = String.format("Failed to process the identities, could not properly open the KDC operation handler: %s",
+ e.getMessage());
+ actionLog.writeStdErr(message);
+ LOG.error(message);
+ throw new AmbariException(message, e);
+ }
- // Create the data file reader to parse and iterate through the records
- KerberosIdentityDataFileReader reader = null;
- try {
- reader = kerberosIdentityDataFileReaderFactory.createKerberosIdentityDataFileReader(identityDataFile);
- for (Map<String, String> record : reader) {
- // Process the current record
- commandReport = processRecord(record, defaultRealm, handler, kerberosConfiguration, requestSharedDataContext);
-
- // If the principal processor returns a CommandReport, than it is time to stop since
- // an error condition has probably occurred, else all is assumed to be well.
- if (commandReport != null) {
- break;
- }
- }
- } catch (AmbariException e) {
- // Catch this separately from IOException since the reason it was thrown was not the same
- // Note: AmbariException is an IOException, so there may be some confusion
- throw new AmbariException(e.getMessage(), e);
- } catch (IOException e) {
- String message = String.format("Failed to process the identities, cannot read the index file: %s",
- identityDataFile.getAbsolutePath());
- actionLog.writeStdErr(message);
- LOG.error(message, e);
- throw new AmbariException(message, e);
- } finally {
- if (reader != null) {
- // The reader needs to be closed, if it fails to close ignore the exception since
- // there is little we can or care to do about it now.
- try {
- reader.close();
- } catch (IOException e) {
- // Ignore this...
- }
- }
-
- // The KerberosOperationHandler needs to be closed, if it fails to close ignore the
- // exception since there is little we can or care to do about it now.
- try {
- handler.close();
- } catch (KerberosOperationException e) {
- // Ignore this...
- }
+ try {
+ for (ResolvedKerberosKeytab rkk : kerberosKeytabController.getFilteredKeytabs((Map<String, Collection<String>>) getServiceComponentFilter(), getHostFilter(), getIdentityFilter())) {
+ for (ResolvedKerberosPrincipal principal : rkk.getPrincipals()) {
+ commandReport = processIdentity(principal, handler, kerberosConfiguration, requestSharedDataContext);
+ // If the principal processor returns a CommandReport, than it is time to stop since
+ // an error condition has probably occurred, else all is assumed to be well.
+ if (commandReport != null) {
+ break;
}
}
}
+ } finally {
+ // The KerberosOperationHandler needs to be closed, if it fails to close ignore the
+ // exception since there is little we can or care to do about it now.
+ try {
+ handler.close();
+ } catch (KerberosOperationException e) {
+ // Ignore this...
+ }
}
+
}
actionLog.writeStdOut("Processing identities completed.");
@@ -502,11 +466,10 @@ public abstract class KerberosServerAction extends AbstractServerAction {
* Processes an identity as necessary.
* <p/>
* This method is called from {@link #processIdentities(Map)} for each
- * identity "record" found in the Kerberos identity metadata file. After processing, it is expected
+ * principal found by specified filter. After processing, it is expected
* that the return value is null on success and a CommandReport (indicating the error) on failure.
*
- * @param identityRecord a Map containing the data for the current identity record
- * @param evaluatedPrincipal a String indicating the relevant principal
+ * @param resolvedPrincipal a ResolvedKerberosPrincipal object to process
* @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
* tasks for specific Kerberos implementations
* (MIT, Active Directory, etc...)
@@ -516,48 +479,12 @@ public abstract class KerberosServerAction extends AbstractServerAction {
* condition; or null, indicating a success condition
* @throws AmbariException if an error occurs while processing the identity record
*/
- protected abstract CommandReport processIdentity(Map<String, String> identityRecord,
- String evaluatedPrincipal,
+ protected abstract CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal,
KerberosOperationHandler operationHandler,
Map<String, String> kerberosConfiguration,
Map<String, Object> requestSharedDataContext)
throws AmbariException;
- /**
- * Process and prepares an identity record to be handled by the implementing class.
- * <p/>
- * Given the data from the record Map, attempts to replace variables in the principal pattern to
- * generate a concrete principal value to further process. This "evaluated principal" is then passed to
- * {@link #processIdentity(Map, String, KerberosOperationHandler, Map, Map)}
- * to be handled as needed.
- *
- * @param record a Map containing the data for the current identity record
- * @param defaultRealm a String declaring the default Kerberos realm
- * @param operationHandler a KerberosOperationHandler used to perform Kerberos-related
- * tasks for specific Kerberos implementations
- * (MIT, Active Directory, etc...)
- * @param kerberosConfiguration a Map of configuration properties from kerberos-env
- * @param requestSharedDataContext a Map to be used a shared data among all ServerActions related
- * to a given request @return a CommandReport, indicating an error
- * condition; or null, indicating a success condition
- * @throws AmbariException if an error occurs while processing the identity record
- */
- private CommandReport processRecord(Map<String, String> record, String defaultRealm,
- KerberosOperationHandler operationHandler,
- Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext)
- throws AmbariException {
- CommandReport commandReport = null;
-
- if (record != null) {
- String principal = record.get(KerberosIdentityDataFileReader.PRINCIPAL);
- if (principal != null) {
- commandReport = processIdentity(record, principal, operationHandler, kerberosConfiguration, requestSharedDataContext);
- }
- }
-
- return commandReport;
- }
-
protected void deleteDataDirectory(String dataDirectoryPath) {
// Make sure this is a relevant directory. We don't want to accidentally allow _ANY_ directory
// to be deleted.
@@ -600,7 +527,32 @@ public abstract class KerberosServerAction extends AbstractServerAction {
return hostFilers != null && hostFilers.size() > 0;
}
- protected Long ambariServerHostID(){
+
+ protected Map<String, ? extends Collection<String>> getServiceComponentFilter() {
+ String serializedValue = getCommandParameterValue(SERVICE_COMPONENT_FILTER);
+
+ if (serializedValue != null) {
+ Type type = new TypeToken<Map<String, ? extends Collection<String>>>() {
+ }.getType();
+ return StageUtils.getGson().fromJson(serializedValue, type);
+ } else {
+ return null;
+ }
+ }
+
+ protected Collection<String> getIdentityFilter() {
+ String serializedValue = getCommandParameterValue(IDENTITY_FILTER);
+
+ if (serializedValue != null) {
+ Type type = new TypeToken<Collection<String>>() {
+ }.getType();
+ return StageUtils.getGson().fromJson(serializedValue, type);
+ } else {
+ return null;
+ }
+ }
+
+ protected Long ambariServerHostID() {
String ambariServerHostName = StageUtils.getHostName();
HostEntity ambariServerHostEntity = hostDAO.findByName(ambariServerHostName);
return (ambariServerHostEntity == null)
@@ -608,6 +560,65 @@ public abstract class KerberosServerAction extends AbstractServerAction {
: ambariServerHostEntity.getHostId();
}
+
+ public static class KerberosCommandParameters {
+ private Map<String, String> params;
+
+ public KerberosCommandParameters(ExecutionCommand ec) {
+ params = ec.getCommandParams();
+ }
+
+ public KerberosCommandParameters(AbstractServerAction serverAction) {
+ this(serverAction.getExecutionCommand());
+ }
+
+ public Set<String> getHostFilter() {
+ String serializedValue = getCommandParameterValue(HOST_FILTER);
+
+ if (serializedValue != null) {
+ Type type = new TypeToken<Set<String>>() {
+ }.getType();
+ return StageUtils.getGson().fromJson(serializedValue, type);
+ } else {
+ return null;
+ }
+ }
+
+ public boolean hasHostFilters() {
+ Set<String> hostFilers = getHostFilter();
+ return hostFilers != null && hostFilers.size() > 0;
+ }
+
+ public Map<String, ? extends Collection<String>> getServiceComponentFilter() {
+ String serializedValue = getCommandParameterValue(SERVICE_COMPONENT_FILTER);
+
+ if (serializedValue != null) {
+ Type type = new TypeToken<Map<String, ? extends Collection<String>>>() {
+ }.getType();
+ return StageUtils.getGson().fromJson(serializedValue, type);
+ } else {
+ return null;
+ }
+ }
+
+ public Collection<String> getIdentityFilter() {
+ String serializedValue = getCommandParameterValue(IDENTITY_FILTER);
+
+ if (serializedValue != null) {
+ Type type = new TypeToken<Collection<String>>() {
+ }.getType();
+ return StageUtils.getGson().fromJson(serializedValue, type);
+ } else {
+ return null;
+ }
+ }
+
+ public String getCommandParameterValue(String propertyName) {
+ Map<String, String> commandParameters = params;
+ return (commandParameters == null) ? null : commandParameters.get(propertyName);
+ }
+ }
+
/**
* A Kerberos operation type
* <ul>
@@ -623,7 +634,7 @@ public abstract class KerberosServerAction extends AbstractServerAction {
RECREATE_ALL,
/**
- * Generate keytabs for only those that are missing
+ * Generate keytabs for only those that are missing
*/
CREATE_MISSING,
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
index 671ad95..2d29bdc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
@@ -29,11 +29,11 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,17 +70,7 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities
Map<String, String> commandParameters = getCommandParameters();
- String preconfigureServices = getCommandParameterValue(commandParameters, PRECONFIGURE_SERVICES);
- PreconfigureServiceType type = null;
- if (!StringUtils.isEmpty(preconfigureServices)) {
- try {
- type = PreconfigureServiceType.valueOf(preconfigureServices.toUpperCase());
- } catch (Throwable t) {
- LOG.warn("Invalid preconfigure_services value, assuming DEFAULT: {}", preconfigureServices);
- type = PreconfigureServiceType.DEFAULT;
- }
- }
-
+ PreconfigureServiceType type = getCommandPreconfigureType();
KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster, type != PreconfigureServiceType.NONE);
if (type == PreconfigureServiceType.ALL) {
// Force all services to be flagged for pre-configuration...
@@ -144,7 +134,7 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities
}
@Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext) throws AmbariException {
+ protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, Map<String, Object> requestSharedDataContext) throws AmbariException {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
index 83a2106..c7f2003 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
@@ -140,15 +140,6 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber
return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
}
- @Override
- protected CommandReport processIdentity(Map<String, String> identityRecord, String evaluatedPrincipal,
- KerberosOperationHandler operationHandler,
- Map<String, String> kerberosConfiguration,
- Map<String, Object> requestSharedDataContext)
- throws AmbariException {
- throw new UnsupportedOperationException();
- }
-
/**
* Calls {@link KerberosHelper#getKerberosDescriptor(Cluster, boolean)}
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
new file mode 100644
index 0000000..4993902
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.kerberos.stageutils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
+import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
+import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * Helper class to construct convenient wrappers around database entities related to kerberos.
+ */
+@Singleton
+public class KerberosKeytabController {
+ @Inject
+ private KerberosKeytabDAO kerberosKeytabDAO;
+
+ @Inject
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
+ /**
+ * Tries to find keytab by keytab path in destination filesystem.
+ *
+ * @param file keytab path
+ * @return found keytab or null
+ */
+ public ResolvedKerberosKeytab getKeytabByFile(String file) {
+ return getKeytabByFile(file, true);
+ }
+
+ /**
+ * Tries to find keytab by keytab path in destination filesystem.
+ *
+ * @param file keytab path
+ * @param resolvePrincipals include resolved principals
+ * @return found keytab or null
+ */
+ public ResolvedKerberosKeytab getKeytabByFile(String file, boolean resolvePrincipals) {
+ return fromKeytabEntity(kerberosKeytabDAO.find(file), resolvePrincipals);
+ }
+
+ /**
+ * Returns all keytabs managed by ambari.
+ *
+ * @return all keytabs
+ */
+ public Set<ResolvedKerberosKeytab> getAllKeytabs() {
+ return fromKeytabEntities(kerberosKeytabDAO.findAll());
+ }
+
+ /**
+ * Returns all keytabs that contains given principal.
+ *
+ * @param rkp principal to filter keytabs by
+ * @return set of keytabs found
+ */
+ public Set<ResolvedKerberosKeytab> getFromPrincipal(ResolvedKerberosPrincipal rkp) {
+ return fromKeytabEntities(kerberosKeytabDAO.findByPrincipalAndHost(rkp.getPrincipal(), rkp.getHostId()));
+ }
+
+ /**
+ * Returns keytabs with principals filtered by host, principal name or service(and component) names.
+ *
+ * @param serviceComponentFilter service-component filter
+ * @param hostFilter host filter
+ * @param identityFilter identity(principal) filter
+ * @return set of keytabs found
+ */
+ public Set<ResolvedKerberosKeytab> getFilteredKeytabs(Map<String, Collection<String>> serviceComponentFilter,
+ Set<String> hostFilter, Collection<String> identityFilter) {
+ if (serviceComponentFilter == null && hostFilter == null && identityFilter == null) {
+ return getAllKeytabs();
+ }
+ List<KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter> filters = splitServiceFilter(serviceComponentFilter);
+ for (KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter filter : filters) {
+ filter.setHostNames(hostFilter);
+ filter.setPrincipals(identityFilter);
+ }
+
+ Set<ResolvedKerberosPrincipal> filteredPrincipals = fromPrincipalEntities(kerberosKeytabPrincipalDAO.findByFilters(filters));
+ HashMap<String, ResolvedKerberosKeytab> resultMap = new HashMap<>();
+ for (ResolvedKerberosPrincipal principal : filteredPrincipals) {
+ if (!resultMap.containsKey(principal.getKeytabPath())) {
+ resultMap.put(principal.getKeytabPath(), getKeytabByFile(principal.getKeytabPath(), false));
+ }
+ ResolvedKerberosKeytab keytab = resultMap.get(principal.getKeytabPath());
+ keytab.addPrincipal(principal);
+ }
+ return Sets.newHashSet(resultMap.values());
+ }
+
+ /**
+ * This function split serviceComponentFilter to two filters, one with specific components, and another one with service
+ * only. Can return only one filter if filter contain only one type of mapping(whole service or component based)
+ * or empty filter if no serviceComponentFilter provided.
+ *
+ * @param serviceComponentFilter
+ * @return
+ */
+ private List<KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter> splitServiceFilter(Map<String, Collection<String>> serviceComponentFilter) {
+ if (serviceComponentFilter != null && serviceComponentFilter.size() > 0) {
+ Set<String> serviceSet = new HashSet<>();
+ Set<String> componentSet = new HashSet<>();
+ Set<String> serviceOnlySet = new HashSet<>();
+ serviceSet.addAll(serviceComponentFilter.keySet());
+ for (String serviceName : serviceSet) {
+ Collection<String> serviceComponents = serviceComponentFilter.get(serviceName);
+ if (serviceComponents.contains("*")) { // star means that this is filtered by whole SERVICE
+ serviceOnlySet.add(serviceName);
+ serviceSet.remove(serviceName); // remove service from regular
+ } else {
+ componentSet.addAll(serviceComponents);
+ }
+ }
+ List<KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter> result = new ArrayList<>();
+ if (serviceSet.size() > 0) {
+ result.add(new KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter(
+ null,
+ serviceSet,
+ componentSet,
+ null
+ ));
+ }
+ if (serviceOnlySet.size() > 0) {
+ result.add(new KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter(
+ null,
+ serviceOnlySet,
+ null,
+ null
+ ));
+ }
+ if (result.size() > 0) {
+ return result;
+ }
+ }
+
+ return Lists.newArrayList(new KerberosKeytabPrincipalDAO.KerberosKeytabPrincipalFilter(null,null,null,null));
+ }
+
+ private ResolvedKerberosKeytab fromKeytabEntity(KerberosKeytabEntity kke, boolean resolvePrincipals) {
+ Set<ResolvedKerberosPrincipal> principals = resolvePrincipals ? fromPrincipalEntities(kke.getKerberosKeytabPrincipalEntities()) : new HashSet<>();
+ return new ResolvedKerberosKeytab(
+ kke.getKeytabPath(),
+ kke.getOwnerName(),
+ kke.getOwnerAccess(),
+ kke.getGroupName(),
+ kke.getGroupAccess(),
+ principals,
+ kke.isAmbariServerKeytab(),
+ kke.isWriteAmbariJaasFile()
+ );
+ }
+
+ private ResolvedKerberosKeytab fromKeytabEntity(KerberosKeytabEntity kke) {
+ return fromKeytabEntity(kke, true);
+ }
+
+ private Set<ResolvedKerberosKeytab> fromKeytabEntities(Collection<KerberosKeytabEntity> keytabEntities) {
+ ImmutableSet.Builder<ResolvedKerberosKeytab> builder = ImmutableSet.builder();
+ for (KerberosKeytabEntity kkpe : keytabEntities) {
+ builder.add(fromKeytabEntity(kkpe));
+ }
+ return builder.build();
+ }
+
+ private Set<ResolvedKerberosPrincipal> fromPrincipalEntities(Collection<KerberosKeytabPrincipalEntity> principalEntities) {
+ ImmutableSet.Builder<ResolvedKerberosPrincipal> builder = ImmutableSet.builder();
+ for (KerberosKeytabPrincipalEntity kkpe : principalEntities) {
+ KerberosPrincipalEntity kpe = kkpe.getPrincipalEntity();
+ ResolvedKerberosPrincipal rkp = new ResolvedKerberosPrincipal(
+ kkpe.getHostId(),
+ kkpe.getHostName(),
+ kkpe.getPrincipalName(),
+ kpe.isService(),
+ kpe.getCachedKeytabPath(),
+ kkpe.getKeytabPath(),
+ kkpe.getServiceMappingAsMultimap());
+ builder.add(rkp);
+ }
+ return builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosKeytab.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosKeytab.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosKeytab.java
index 17e484a..3233915 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosKeytab.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosKeytab.java
@@ -18,21 +18,17 @@
package org.apache.ambari.server.serveraction.kerberos.stageutils;
-import java.util.Map;
+import java.util.HashSet;
import java.util.Set;
import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
-import org.apache.commons.lang3.tuple.Pair;
-
-import com.google.common.collect.ImmutableSet;
/**
* Class that represents keytab. Contains principals that mapped to host.
- * Same keytab can have different set of principals on different hosts.
+ * Same keytab can have different set of principals on different hosts for different services.
+ * Each principal identified by host and keytab it belongs to and contain mapping that shows in which services and
+ * components given principal used.
*/
-// TODO This class need to replace {@link org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFile}
-// TODO and all related structures and become main item that {@link org.apache.ambari.server.serveraction.kerberos.KerberosServerAction}
-// TODO operates with instead of identity records.
public class ResolvedKerberosKeytab {
private String ownerName = null;
@@ -40,43 +36,36 @@ public class ResolvedKerberosKeytab {
private String groupName = null;
private String groupAccess = null;
private String file = null;
- private Set<Pair<Long, Pair<String, String>>> mappedPrincipals = null;
+ private Set<ResolvedKerberosPrincipal> principals = new HashSet<>();
private boolean isAmbariServerKeytab = false;
private boolean mustWriteAmbariJaasFile = false;
public ResolvedKerberosKeytab(
- String file,
- String ownerName,
- String ownerAccess,
- String groupName,
- String groupAccess,
- Set<Pair<Long, Pair<String, String>>> mappedPrincipals,
- boolean isAmbariServerKeytab,
- boolean writeAmbariJaasFile
+ String file,
+ String ownerName,
+ String ownerAccess,
+ String groupName,
+ String groupAccess,
+ Set<ResolvedKerberosPrincipal> principals,
+ boolean isAmbariServerKeytab,
+ boolean writeAmbariJaasFile
) {
this.ownerName = ownerName;
this.ownerAccess = ownerAccess;
this.groupName = groupName;
this.groupAccess = groupAccess;
this.file = file;
- this.mappedPrincipals = mappedPrincipals;
+ setPrincipals(principals);
this.isAmbariServerKeytab = isAmbariServerKeytab;
this.mustWriteAmbariJaasFile = writeAmbariJaasFile;
+
}
/**
* Gets the path to the keytab file
- * <p/>
- * The value may include variable placeholders to be replaced as needed
- * <ul>
- * <li>
- * ${variable} placeholders are replaced on the server - see
- * {@link VariableReplacementHelper#replaceVariables(String, Map)}
- * </li>
- * </ul>
*
* @return a String declaring the keytab file's absolute path
- * @see VariableReplacementHelper#replaceVariables(String, Map)
+ * @see VariableReplacementHelper#replaceVariables(String, java.util.Map)
*/
public String getFile() {
return file;
@@ -175,47 +164,36 @@ public class ResolvedKerberosKeytab {
/**
* Gets evaluated host-to-principal set associated with given keytab.
*
- * @return a Set with mappedPrincipals associated with given keytab
+ * @return a Set with principals associated with given keytab
*/
- public Set<Pair<Long, Pair<String, String>>> getMappedPrincipals() {
- return mappedPrincipals;
+ public Set<ResolvedKerberosPrincipal> getPrincipals() {
+ return principals;
}
/**
* Sets evaluated host-to-principal set associated with given keytab.
*
- * @param mappedPrincipals a Map with host-to-principal mapping associated with given keytab
- */
- public void setMappedPrincipals(Set<Pair<Long, Pair<String, String>>> mappedPrincipals) {
- this.mappedPrincipals = mappedPrincipals;
- }
-
- /**
- * Gets set of hosts associated with given keytab.
- *
- * @return a Set with hosts
+ * @param principals set of principals to add
*/
- public Set<Long> getHosts() {
- ImmutableSet.Builder<Long> builder = ImmutableSet.builder();
- for (Pair<Long, Pair<String, String>> principal : getMappedPrincipals()) {
- if (principal.getLeft() != null) {
- builder.add(principal.getLeft());
+ public void setPrincipals(Set<ResolvedKerberosPrincipal> principals) {
+ this.principals = principals;
+ if (principals != null) {
+ for (ResolvedKerberosPrincipal principal : this.principals) {
+ principal.setResolvedKerberosKeytab(this);
}
}
- return builder.build();
}
/**
- * Gets a set of principals associated with given keytab.
+ * Add principal to keytab.
*
- * @return a Set of principals
+ * @param principal resolved principal to add
*/
- public Set<Pair<String, String>> getPrincipals() {
- ImmutableSet.Builder<Pair<String, String>> builder = ImmutableSet.builder();
- for (Pair<Long, Pair<String, String>> principal : getMappedPrincipals()) {
- builder.add(principal.getRight());
+ public void addPrincipal(ResolvedKerberosPrincipal principal) {
+ if (!principals.contains(principal)) {
+ principal.setResolvedKerberosKeytab(this);
+ principals.add(principal);
}
- return builder.build();
}
/**
@@ -254,4 +232,37 @@ public class ResolvedKerberosKeytab {
public void setMustWriteAmbariJaasFile(boolean mustWriteAmbariJaasFile) {
this.mustWriteAmbariJaasFile = mustWriteAmbariJaasFile;
}
+
+ /**
+ * Merge principals from one keytab to given.
+ *
+ * @param otherKeytab keytab to merge principals from
+ */
+ public void mergePrincipals(ResolvedKerberosKeytab otherKeytab) {
+ for (ResolvedKerberosPrincipal rkp : otherKeytab.getPrincipals()) {
+ ResolvedKerberosPrincipal existent = findPrincipal(rkp.getHostId(), rkp.getPrincipal(), rkp.getKeytabPath());
+ if (existent != null) {
+ existent.mergeComponentMapping(rkp);
+ } else {
+ principals.add(rkp);
+ }
+ }
+ }
+
+ private ResolvedKerberosPrincipal findPrincipal(Long hostId, String principal, String keytabPath) {
+ for (ResolvedKerberosPrincipal rkp : principals) {
+ boolean hostIdIsSame;
+ if(hostId != null && rkp.getHostId() != null){
+ hostIdIsSame = hostId.equals(rkp.getHostId());
+ } else if(hostId == null && rkp.getHostId() == null) {
+ hostIdIsSame = true;
+ } else {
+ hostIdIsSame = false;
+ }
+ if (hostIdIsSame && principal.equals(rkp.getPrincipal())&& keytabPath.equals(rkp.getKeytabPath())) {
+ return rkp;
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosPrincipal.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosPrincipal.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosPrincipal.java
new file mode 100644
index 0000000..100c1e2
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/ResolvedKerberosPrincipal.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.ambari.server.serveraction.kerberos.stageutils;
+
+import org.apache.ambari.server.utils.StageUtils;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+/**
+ * Class that represents principal and it info(host, keytab path, service and component mapping).
+ */
+public class ResolvedKerberosPrincipal {
+ private Long hostId;
+ private String hostName;
+ private String principal;
+ private boolean isService;
+ private String cacheFile;
+ private Multimap<String, String> serviceMapping = ArrayListMultimap.create();
+ private String keytabPath;
+ private ResolvedKerberosKeytab resolvedKerberosKeytab;
+
+ public ResolvedKerberosPrincipal(Long hostId, String hostName, String principal, boolean isService, String cacheFile, String serviceName, String componentName, String keytabPath) {
+ this.hostId = hostId;
+ this.hostName = hostName;
+ this.principal = principal;
+ this.isService = isService;
+ this.cacheFile = cacheFile;
+ this.keytabPath = keytabPath;
+ addComponentMapping(serviceName, componentName);
+ }
+
+ public ResolvedKerberosPrincipal(Long hostId, String hostName, String principal, boolean isService, String cacheFile, String keytabPath) {
+ this.hostId = hostId;
+ this.hostName = hostName;
+ this.principal = principal;
+ this.isService = isService;
+ this.cacheFile = cacheFile;
+ this.keytabPath = keytabPath;
+ }
+
+ public ResolvedKerberosPrincipal(Long hostId, String hostName, String principal, boolean isService, String cacheFile, String keytabPath, Multimap<String, String> serviceMapping) {
+ this.hostId = hostId;
+ this.hostName = hostName;
+ this.principal = principal;
+ this.isService = isService;
+ this.cacheFile = cacheFile;
+ this.keytabPath = keytabPath;
+ this.serviceMapping = serviceMapping;
+ }
+
+ public void addComponentMapping(String serviceName, String componentName) {
+ if (serviceName == null){
+ serviceName = "";
+ }
+ if (componentName == null) {
+ componentName = "*";
+ }
+ serviceMapping.get(serviceName).add(componentName);
+ }
+
+ public void mergeComponentMapping(ResolvedKerberosPrincipal other) {
+ serviceMapping.putAll(other.getServiceMapping());
+ }
+
+ public String getKeytabPath() {
+ return keytabPath;
+ }
+
+ public void setKeytabPath(String keytabPath) {
+ this.keytabPath = keytabPath;
+ }
+
+ public Long getHostId() {
+ return hostId;
+ }
+
+ public void setHostId(Long hostId) {
+ this.hostId = hostId;
+ }
+
+ public String getHostName() {
+ if (hostName == null) {
+ return StageUtils.getHostName();
+ }
+ return hostName;
+ }
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public boolean isService() {
+ return isService;
+ }
+
+ public void setService(boolean service) {
+ isService = service;
+ }
+
+ public String getCacheFile() {
+ return cacheFile;
+ }
+
+ public void setCacheFile(String cacheFile) {
+ this.cacheFile = cacheFile;
+ }
+
+ public Multimap<String, String> getServiceMapping() {
+ return serviceMapping;
+ }
+
+ public void setServiceMapping(Multimap<String, String> serviceMapping) {
+ this.serviceMapping = serviceMapping;
+ }
+
+ public ResolvedKerberosKeytab getResolvedKerberosKeytab() {
+ return resolvedKerberosKeytab;
+ }
+
+ public void setResolvedKerberosKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab) {
+ this.resolvedKerberosKeytab = resolvedKerberosKeytab;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ResolvedKerberosPrincipal principal1 = (ResolvedKerberosPrincipal) o;
+ return isService == principal1.isService &&
+ Objects.equal(hostId, principal1.hostId) &&
+ Objects.equal(hostName, principal1.hostName) &&
+ Objects.equal(principal, principal1.principal) &&
+ Objects.equal(cacheFile, principal1.cacheFile) &&
+ Objects.equal(serviceMapping, principal1.serviceMapping) &&
+ Objects.equal(keytabPath, principal1.keytabPath);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(hostId, hostName, principal, isService, cacheFile, serviceMapping, keytabPath);
+ }
+}
[02/51] [abbrv] ambari git commit: AMBARI-22639. Log Feeder refactor:
integrate with spring boot (oleewere)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
index 8f8c4fd..e8066be 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputConfigUploader.java
@@ -27,60 +27,76 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
+import org.apache.ambari.logfeeder.common.ConfigHandler;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
+import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
import org.apache.ambari.logsearch.config.api.LogSearchConfigLogFeeder;
-import org.apache.log4j.Logger;
import com.google.common.io.Files;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
public class InputConfigUploader extends Thread {
- protected static final Logger LOG = Logger.getLogger(InputConfigUploader.class);
+
+ protected static final Logger LOG = LoggerFactory.getLogger(InputConfigUploader.class);
private static final long SLEEP_BETWEEN_CHECK = 2000;
- private final File configDir;
- private final FilenameFilter inputConfigFileFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith("input.config-") && name.endsWith(".json");
- }
- };
+ private File configDir;
+ private final FilenameFilter inputConfigFileFilter = (dir, name) -> name.startsWith("input.config-") && name.endsWith(".json");
private final Set<String> filesHandled = new HashSet<>();
private final Pattern serviceNamePattern = Pattern.compile("input.config-(.+).json");
- private final LogSearchConfigLogFeeder config;
-
- public static void load(LogSearchConfigLogFeeder config) {
- new InputConfigUploader(config).start();
- }
-
- private InputConfigUploader(LogSearchConfigLogFeeder config) {
+
+ @Inject
+ private LogSearchConfigLogFeeder config;
+
+ @Inject
+ private LogFeederProps logFeederProps;
+
+ @Inject
+ private LogLevelFilterHandler logLevelFilterHandler;
+
+ @Inject
+ private ConfigHandler configHandler;
+
+ public InputConfigUploader() {
super("Input Config Loader");
setDaemon(true);
-
- this.configDir = new File(LogFeederPropertiesUtil.getConfigDir());
- this.config = config;
+ }
+
+ @PostConstruct
+ public void init() throws Exception {
+ this.configDir = new File(logFeederProps.getConfDir());
+ this.start();
+ config.monitorInputConfigChanges(configHandler, logLevelFilterHandler, logFeederProps.getClusterName());
}
@Override
public void run() {
while (true) {
File[] inputConfigFiles = configDir.listFiles(inputConfigFileFilter);
- for (File inputConfigFile : inputConfigFiles) {
- if (!filesHandled.contains(inputConfigFile.getAbsolutePath())) {
- try {
- Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
- m.find();
- String serviceName = m.group(1);
- String inputConfig = Files.toString(inputConfigFile, Charset.defaultCharset());
-
- if (!config.inputConfigExists(serviceName)) {
- config.createInputConfig(LogFeederPropertiesUtil.getClusterName(), serviceName, inputConfig);
+ if (inputConfigFiles != null) {
+ for (File inputConfigFile : inputConfigFiles) {
+ if (!filesHandled.contains(inputConfigFile.getAbsolutePath())) {
+ try {
+ Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
+ m.find();
+ String serviceName = m.group(1);
+ String inputConfig = Files.toString(inputConfigFile, Charset.defaultCharset());
+ if (!config.inputConfigExists(serviceName)) {
+ config.createInputConfig(logFeederProps.getClusterName(), serviceName, inputConfig);
+ }
+ filesHandled.add(inputConfigFile.getAbsolutePath());
+ } catch (Exception e) {
+ LOG.warn("Error handling file " + inputConfigFile.getAbsolutePath(), e);
}
- filesHandled.add(inputConfigFile.getAbsolutePath());
- } catch (Exception e) {
- LOG.warn("Error handling file " + inputConfigFile.getAbsolutePath(), e);
}
}
+ } else {
+ LOG.warn("Cannot find input config files in config dir ({})", logFeederProps.getConfDir());
}
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
index f1b422f..d1f38ed 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
@@ -33,22 +33,25 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.FileUtil;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.solr.common.util.Base64;
+import javax.inject.Inject;
+
public class InputManager {
private static final Logger LOG = Logger.getLogger(InputManager.class);
private static final String CHECKPOINT_SUBFOLDER_NAME = "logfeeder_checkpoints";
private Map<String, List<Input>> inputs = new HashMap<>();
- private Set<Input> notReadyList = new HashSet<Input>();
+ private Set<Input> notReadyList = new HashSet<>();
private boolean isDrain = false;
@@ -59,6 +62,9 @@ public class InputManager {
private Thread inputIsReadyMonitor;
+ @Inject
+ private LogFeederProps logFeederProps;
+
public List<Input> getInputList(String serviceName) {
return inputs.get(serviceName);
}
@@ -118,11 +124,11 @@ public class InputManager {
}
private void initCheckPointSettings() {
- checkPointExtension = LogFeederPropertiesUtil.getCheckPointExtension();
+ checkPointExtension = logFeederProps.getCheckPointExtension();
LOG.info("Determining valid checkpoint folder");
boolean isCheckPointFolderValid = false;
// We need to keep track of the files we are reading.
- String checkPointFolder = LogFeederPropertiesUtil.getCheckpointFolder();
+ String checkPointFolder = logFeederProps.getCheckpointFolder();
if (!StringUtils.isEmpty(checkPointFolder)) {
checkPointFolderFile = new File(checkPointFolder);
isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
@@ -130,8 +136,7 @@ public class InputManager {
if (!isCheckPointFolderValid) {
// Let's use tmp folder
- String tmpFolder = LogFeederPropertiesUtil.getLogFeederTempDir();
- checkPointFolderFile = new File(tmpFolder, CHECKPOINT_SUBFOLDER_NAME);
+ checkPointFolderFile = new File(logFeederProps.getTmpDir(), CHECKPOINT_SUBFOLDER_NAME);
LOG.info("Checking if tmp folder can be used for checkpoints. Folder=" + checkPointFolderFile);
isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
if (isCheckPointFolderValid) {
@@ -184,7 +189,7 @@ public class InputManager {
public void startInputs(String serviceName) {
for (Input input : inputs.get(serviceName)) {
try {
- input.init();
+ input.init(logFeederProps);
if (input.isReady()) {
input.monitor();
} else {
@@ -419,4 +424,13 @@ public class InputManager {
}
}
}
+
+ @VisibleForTesting
+ public void setLogFeederProps(LogFeederProps logFeederProps) {
+ this.logFeederProps = logFeederProps;
+ }
+
+ public LogFeederProps getLogFeederProps() {
+ return logFeederProps;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
index df6c941..2b2d145 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
@@ -30,10 +30,11 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ambari.logfeeder.conf.InputSimulateConfig;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.filter.FilterJSON;
import org.apache.ambari.logfeeder.output.Output;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl;
@@ -65,32 +66,37 @@ public class InputSimulate extends Input {
}
private final Random random = new Random(System.currentTimeMillis());
-
- private final List<String> types;
- private final String level;
- private final int numberOfWords;
- private final int minLogWords;
- private final int maxLogWords;
- private final long sleepMillis;
- private final String host;
-
- public InputSimulate() throws Exception {
+
+ private InputSimulateConfig conf;
+ private List<String> types;
+ private String level;
+ private int numberOfWords;
+ private int minLogWords;
+ private int maxLogWords;
+ private long sleepMillis;
+ private String host;
+
+ @Override
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
+ conf = logFeederProps.getInputSimulateConfig();
this.types = getSimulatedLogTypes();
- this.level = LogFeederPropertiesUtil.getSimulateLogLevel();
- this.numberOfWords = LogFeederPropertiesUtil.getSimulateNumberOfWords();
- this.minLogWords = LogFeederPropertiesUtil.getSimulateMinLogWords();
- this.maxLogWords = LogFeederPropertiesUtil.getSimulateMaxLogWords();
- this.sleepMillis = LogFeederPropertiesUtil.getSimulateSleepMilliseconds();
+ this.level = conf.getSimulateLogLevel();
+ this.numberOfWords = conf.getSimulateNumberOfWords();
+ this.minLogWords = conf.getSimulateMinLogWords();
+ this.maxLogWords = conf.getSimulateMaxLogWords();
+ this.sleepMillis = conf.getSimulateSleepMilliseconds();
this.host = "#" + hostNumber.incrementAndGet() + "-" + LogFeederUtil.hostName;
-
+
Filter filter = new FilterJSON();
filter.loadConfig(new FilterJsonDescriptorImpl());
filter.setInput(this);
addFilter(filter);
+
}
-
+
private List<String> getSimulatedLogTypes() {
- String logsToSimulate = LogFeederPropertiesUtil.getSimulateLogIds();
+ String logsToSimulate = conf.getSimulateLogIds();
return (logsToSimulate == null) ?
inputTypes :
Arrays.asList(logsToSimulate.split(","));
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/FilterLogData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/FilterLogData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/FilterLogData.java
deleted file mode 100644
index 6173f53..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/FilterLogData.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.loglevelfilter;
-
-import java.util.Map;
-
-import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Logger;
-
-public enum FilterLogData {
- INSTANCE;
-
- private static final Logger LOG = Logger.getLogger(FilterLogData.class);
-
- private static final boolean DEFAULT_VALUE = true;
-
- public boolean isAllowed(String jsonBlock, InputMarker inputMarker) {
- if (StringUtils.isEmpty(jsonBlock)) {
- return DEFAULT_VALUE;
- }
- Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
- return isAllowed(jsonObj, inputMarker);
- }
-
- public boolean isAllowed(Map<String, Object> jsonObj, InputMarker inputMarker) {
- if ("audit".equals(inputMarker.input.getInputDescriptor().getRowtype()))
- return true;
-
- boolean isAllowed = applyFilter(jsonObj);
- if (!isAllowed) {
- LOG.trace("Filter block the content :" + LogFeederUtil.getGson().toJson(jsonObj));
- }
- return isAllowed;
- }
-
-
- private boolean applyFilter(Map<String, Object> jsonObj) {
- if (MapUtils.isEmpty(jsonObj)) {
- LOG.warn("Output jsonobj is empty");
- return DEFAULT_VALUE;
- }
-
- String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST);
- String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
- String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
- if (StringUtils.isNotBlank(hostName) && StringUtils.isNotBlank(logId) && StringUtils.isNotBlank(level)) {
- return LogLevelFilterHandler.isAllowed(hostName, logId, level);
- } else {
- return DEFAULT_VALUE;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
index e44873b..83c293b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
@@ -21,7 +21,6 @@ package org.apache.ambari.logfeeder.loglevelfilter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
-import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -29,19 +28,28 @@ import java.util.Map;
import java.util.TimeZone;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
+import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.LogLevelFilterMonitor;
import org.apache.ambari.logsearch.config.api.LogSearchConfig;
import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
public class LogLevelFilterHandler implements LogLevelFilterMonitor {
- private static final Logger LOG = Logger.getLogger(LogLevelFilterHandler.class);
+ private static final Logger LOG = LoggerFactory.getLogger(LogLevelFilterHandler.class);
private static final String TIMEZONE = "GMT";
private static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+
+ private static final boolean DEFAULT_VALUE = true;
private static ThreadLocal<DateFormat> formatter = new ThreadLocal<DateFormat>() {
protected DateFormat initialValue() {
@@ -50,16 +58,19 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
return dateFormat;
}
};
-
- private static LogSearchConfig config;
- private static boolean filterEnabled;
- private static List<String> defaultLogLevels;
- private static Map<String, LogLevelFilter> filters = new HashMap<>();
-
- public static void init(LogSearchConfig config_) {
- config = config_;
- filterEnabled = LogFeederPropertiesUtil.isLogFilterEnabled();
- defaultLogLevels = Arrays.asList(LogFeederPropertiesUtil.getIncludeDefaultLevel().split(","));
+
+ @Inject
+ private LogFeederProps logFeederProps;
+
+ private LogSearchConfig config;
+ private Map<String, LogLevelFilter> filters = new HashMap<>();
+
+ public LogLevelFilterHandler(LogSearchConfig config) {
+ this.config = config;
+ }
+
+ @PostConstruct
+ public void init() {
TimeZone.setDefault(TimeZone.getTimeZone(TIMEZONE));
}
@@ -77,8 +88,8 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
}
}
- public static boolean isAllowed(String hostName, String logId, String level) {
- if (!filterEnabled) {
+ public boolean isAllowed(String hostName, String logId, String level) {
+ if (!logFeederProps.isLogLevelFilterEnabled()) {
return true;
}
@@ -87,7 +98,43 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
return allowedLevels.isEmpty() || allowedLevels.contains(level);
}
- private static synchronized LogLevelFilter findLogFilter(String logId) {
+ public boolean isAllowed(String jsonBlock, InputMarker inputMarker) {
+ if (org.apache.commons.lang3.StringUtils.isEmpty(jsonBlock)) {
+ return DEFAULT_VALUE;
+ }
+ Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
+ return isAllowed(jsonObj, inputMarker);
+ }
+
+ public boolean isAllowed(Map<String, Object> jsonObj, InputMarker inputMarker) {
+ if ("audit".equals(inputMarker.input.getInputDescriptor().getRowtype()))
+ return true;
+
+ boolean isAllowed = applyFilter(jsonObj);
+ if (!isAllowed) {
+ LOG.trace("Filter block the content :" + LogFeederUtil.getGson().toJson(jsonObj));
+ }
+ return isAllowed;
+ }
+
+
+ public boolean applyFilter(Map<String, Object> jsonObj) {
+ if (MapUtils.isEmpty(jsonObj)) {
+ LOG.warn("Output jsonobj is empty");
+ return DEFAULT_VALUE;
+ }
+
+ String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST);
+ String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
+ String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
+ if (org.apache.commons.lang3.StringUtils.isNotBlank(hostName) && org.apache.commons.lang3.StringUtils.isNotBlank(logId) && org.apache.commons.lang3.StringUtils.isNotBlank(level)) {
+ return isAllowed(hostName, logId, level);
+ } else {
+ return DEFAULT_VALUE;
+ }
+ }
+
+ private synchronized LogLevelFilter findLogFilter(String logId) {
LogLevelFilter logFilter = filters.get(logId);
if (logFilter != null) {
return logFilter;
@@ -96,10 +143,10 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
LOG.info("Filter is not present for log " + logId + ", creating default filter");
LogLevelFilter defaultFilter = new LogLevelFilter();
defaultFilter.setLabel(logId);
- defaultFilter.setDefaultLevels(defaultLogLevels);
+ defaultFilter.setDefaultLevels(logFeederProps.getIncludeDefaultLogLevels());
try {
- config.createLogLevelFilter(LogFeederPropertiesUtil.getClusterName(), logId, defaultFilter);
+ config.createLogLevelFilter(logFeederProps.getClusterName(), logId, defaultFilter);
filters.put(logId, defaultFilter);
} catch (Exception e) {
LOG.warn("Could not persist the default filter for log " + logId, e);
@@ -108,7 +155,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
return defaultFilter;
}
- private static List<String> getAllowedLevels(String hostName, LogLevelFilter componentFilter) {
+ private List<String> getAllowedLevels(String hostName, LogLevelFilter componentFilter) {
String componentName = componentFilter.getLabel();
List<String> hosts = componentFilter.getHosts();
List<String> defaultLevels = componentFilter.getDefaultLevels();
@@ -134,7 +181,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
return defaultLevels;
}
- private static boolean isFilterExpired(LogLevelFilter logLevelFilter) {
+ private boolean isFilterExpired(LogLevelFilter logLevelFilter) {
if (logLevelFilter == null)
return false;
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
index c832358..ba986c7 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
@@ -19,15 +19,13 @@
package org.apache.ambari.logfeeder.metrics;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
-import org.apache.ambari.logfeeder.util.SSLUtil;
+import org.apache.ambari.logfeeder.conf.LogFeederSecurityConfig;
+import org.apache.ambari.logfeeder.conf.MetricsCollectorConfig;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
import org.apache.log4j.Logger;
-import com.google.common.base.Splitter;
-
import java.util.Collection;
import java.util.List;
@@ -40,16 +38,16 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink {
private final String collectorPort;
private final String collectorPath;
- public LogFeederAMSClient() {
- String collectorHostsString = LogFeederPropertiesUtil.getMetricsCollectorHosts();
+ public LogFeederAMSClient(MetricsCollectorConfig metricsCollectorConfig, LogFeederSecurityConfig securityConfig) {
+ String collectorHostsString = metricsCollectorConfig.getHostsString();
if (!StringUtils.isBlank(collectorHostsString)) {
collectorHostsString = collectorHostsString.trim();
LOG.info("AMS collector Hosts=" + collectorHostsString);
- collectorHosts = Splitter.on(",").splitToList(collectorHostsString);
- collectorProtocol = LogFeederPropertiesUtil.getMetricsCollectorProtocol();
- collectorPort = LogFeederPropertiesUtil.getMetricsCollectorPort();
- collectorPath = LogFeederPropertiesUtil.getMetricsCollectorPath();
+ collectorHosts = metricsCollectorConfig.getHosts();
+ collectorProtocol = metricsCollectorConfig.getProtocol();
+ collectorPort = metricsCollectorConfig.getPort();
+ collectorPath = metricsCollectorConfig.getPath();
} else {
collectorHosts = null;
collectorProtocol = null;
@@ -57,8 +55,8 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink {
collectorPath = null;
}
- if (StringUtils.isNotBlank(SSLUtil.getTrustStoreLocation())) {
- loadTruststore(SSLUtil.getTrustStoreLocation(), SSLUtil.getTrustStoreType(), SSLUtil.getTrustStorePassword());
+ if (StringUtils.isNotBlank(securityConfig.getTrustStoreLocation())) {
+ loadTruststore(securityConfig.getTrustStoreLocation(), securityConfig.getTrustStoreType(), securityConfig.getTrustStorePassword());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
index 6e8ac04..96084c1 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
@@ -24,11 +24,16 @@ import java.util.HashMap;
import java.util.List;
import java.util.TreeMap;
+import org.apache.ambari.logfeeder.conf.LogFeederSecurityConfig;
+import org.apache.ambari.logfeeder.conf.MetricsCollectorConfig;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
import org.apache.log4j.Logger;
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
public class MetricsManager {
private static final Logger LOG = Logger.getLogger(MetricsManager.class);
@@ -40,13 +45,20 @@ public class MetricsManager {
private int publishIntervalMS = 60 * 1000;
private int maxMetricsBuffer = 60 * 60 * 1000; // If AMS is down, we should not keep the metrics in memory forever
- private HashMap<String, TimelineMetric> metricsMap = new HashMap<String, TimelineMetric>();
+ private HashMap<String, TimelineMetric> metricsMap = new HashMap<>();
private LogFeederAMSClient amsClient = null;
+ @Inject
+ private MetricsCollectorConfig metricsCollectorConfig;
+
+ @Inject
+ private LogFeederSecurityConfig logFeederSecurityConfig;
+
+ @PostConstruct
public void init() {
LOG.info("Initializing MetricsManager()");
if (amsClient == null) {
- amsClient = new LogFeederAMSClient();
+ amsClient = new LogFeederAMSClient(metricsCollectorConfig, logFeederSecurityConfig);
}
if (amsClient.getCollectorUri(null) != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/StatsLogger.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/StatsLogger.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/StatsLogger.java
new file mode 100644
index 0000000..1dd9287
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/StatsLogger.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.metrics;
+
+import org.apache.ambari.logfeeder.common.ConfigHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+
+public class StatsLogger extends Thread {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StatsLogger.class);
+
+ private static final int CHECKPOINT_CLEAN_INTERVAL_MS = 24 * 60 * 60 * 60 * 1000; // 24 hours
+
+ private long lastCheckPointCleanedMS = 0;
+
+ @Inject
+ private ConfigHandler configHandler;
+
+ @Inject
+ private MetricsManager metricsManager;
+
+ public StatsLogger() {
+ super("statLogger");
+ setDaemon(true);
+ }
+
+ @PostConstruct
+ public void init() {
+ this.start();
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Thread.sleep(30 * 1000);
+ } catch (Throwable t) {
+ // Ignore
+ }
+ try {
+ logStats();
+ } catch (Throwable t) {
+ LOG.error("LogStats: Caught exception while logging stats.", t);
+ }
+
+ if (System.currentTimeMillis() > (lastCheckPointCleanedMS + CHECKPOINT_CLEAN_INTERVAL_MS)) {
+ lastCheckPointCleanedMS = System.currentTimeMillis();
+ configHandler.cleanCheckPointFiles();
+ }
+ }
+ }
+
+ private void logStats() {
+ configHandler.logStats();
+ if (metricsManager.isMetricsEnabled()) {
+ List<MetricData> metricsList = new ArrayList<MetricData>();
+ configHandler.addMetrics(metricsList);
+ metricsManager.useMetrics(metricsList);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
index e1a0bb9..4576deb 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.csv.CSVFormat;
@@ -41,8 +42,8 @@ public class OutputFile extends Output {
private String codec;
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
filePath = getStringValue("path");
if (StringUtils.isEmpty(filePath)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
index ba4d60a..ed66eb0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.output;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.spool.LogSpooler;
import org.apache.ambari.logfeeder.output.spool.LogSpoolerContext;
@@ -26,7 +27,6 @@ import org.apache.ambari.logfeeder.output.spool.RolloverCondition;
import org.apache.ambari.logfeeder.output.spool.RolloverHandler;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.LogFeederHDFSUtil;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logfeeder.util.PlaceholderUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
@@ -65,8 +65,8 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
private LogSpooler logSpooler;
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
hdfsOutDir = getStringValue("hdfs_out_dir");
hdfsHost = getStringValue("hdfs_host");
hdfsPort = getStringValue("hdfs_port");
@@ -88,7 +88,7 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
HashMap<String, String> contextParam = buildContextParam();
hdfsOutDir = PlaceholderUtil.replaceVariables(hdfsOutDir, contextParam);
LOG.info("hdfs Output dir=" + hdfsOutDir);
- String localFileDir = LogFeederPropertiesUtil.getLogFeederTempDir() + "hdfs/service/";
+ String localFileDir = logFeederProps.getTmpDir() + "hdfs/service/";
logSpooler = new LogSpooler(localFileDir, filenamePrefix, this, this);
this.startHDFSCopyThread();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
index 52fc6f8..5c8ec82 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
@@ -25,6 +25,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedTransferQueue;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang3.StringUtils;
@@ -66,8 +67,8 @@ public class OutputKafka extends Output {
}
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
Properties props = initProperties();
producer = creteKafkaProducer(props);
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
index 48716fa..f5c4176 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
@@ -26,10 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.loglevelfilter.FilterLogData;
+import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.MurmurHash;
@@ -38,6 +40,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import javax.inject.Inject;
+
public class OutputManager {
private static final Logger LOG = Logger.getLogger(OutputManager.class);
@@ -51,6 +55,12 @@ public class OutputManager {
private static long docCounter = 0;
private MetricData messageTruncateMetric = new MetricData(null, false);
+ @Inject
+ private LogLevelFilterHandler logLevelFilterHandler;
+
+ @Inject
+ private LogFeederProps logFeederProps;
+
private OutputLineFilter outputLineFilter = new OutputLineFilter();
public List<Output> getOutputs() {
@@ -73,7 +83,7 @@ public class OutputManager {
public void init() throws Exception {
for (Output output : outputs) {
- output.init();
+ output.init(logFeederProps);
}
}
@@ -146,7 +156,7 @@ public class OutputManager {
jsonObj.put("message_md5", "" + MurmurHash.hash64A(logMessage.getBytes(), 31174077));
}
}
- if (FilterLogData.INSTANCE.isAllowed(jsonObj, inputMarker)
+ if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker)
&& !outputLineFilter.apply(jsonObj, inputMarker.input)) {
for (Output output : input.getOutputList()) {
try {
@@ -179,7 +189,7 @@ public class OutputManager {
}
public void write(String jsonBlock, InputMarker inputMarker) {
- if (FilterLogData.INSTANCE.isAllowed(jsonBlock, inputMarker)) {
+ if (logLevelFilterHandler.isAllowed(jsonBlock, inputMarker)) {
for (Output output : inputMarker.input.getOutputList()) {
try {
output.write(jsonBlock, inputMarker);
@@ -255,4 +265,22 @@ public class OutputManager {
}
}
}
+
+ public LogLevelFilterHandler getLogLevelFilterHandler() {
+ return logLevelFilterHandler;
+ }
+
+ @VisibleForTesting
+ public void setLogLevelFilterHandler(LogLevelFilterHandler logLevelFilterHandler) {
+ this.logLevelFilterHandler = logLevelFilterHandler;
+ }
+
+ public LogFeederProps getLogFeederProps() {
+ return logFeederProps;
+ }
+
+ @VisibleForTesting
+ public void setLogFeederProps(LogFeederProps logFeederProps) {
+ this.logFeederProps = logFeederProps;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
index 5b213e8..d8eed2b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
@@ -20,13 +20,13 @@ package org.apache.ambari.logfeeder.output;
import com.google.common.annotations.VisibleForTesting;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.spool.LogSpooler;
import org.apache.ambari.logfeeder.output.spool.LogSpoolerContext;
import org.apache.ambari.logfeeder.output.spool.RolloverCondition;
import org.apache.ambari.logfeeder.output.spool.RolloverHandler;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.S3Util;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
@@ -62,8 +62,8 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
private S3Uploader s3Uploader;
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
s3OutputConfiguration = S3OutputConfiguration.fromConfigBlock(this);
}
@@ -206,7 +206,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
@VisibleForTesting
protected LogSpooler createSpooler(String filePath) {
- String spoolDirectory = LogFeederPropertiesUtil.getLogFeederTempDir() + "/s3/service";
+ String spoolDirectory = getLogFeederProps().getTmpDir() + "/s3/service";
LOG.info(String.format("Creating spooler with spoolDirectory=%s, filePath=%s", spoolDirectory, filePath));
return new LogSpooler(spoolDirectory, new File(filePath).getName()+"-", this, this,
s3OutputConfiguration.getRolloverTimeThresholdSecs());
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index 38219df..cdb869a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -35,9 +35,9 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.util.DateUtil;
-import org.apache.ambari.logfeeder.util.LogFeederPropertiesUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputProperties;
import org.apache.ambari.logsearch.config.api.model.outputconfig.OutputSolrProperties;
@@ -109,8 +109,8 @@ public class OutputSolr extends Output implements CollectionStateWatcher {
}
@Override
- public void init() throws Exception {
- super.init();
+ public void init(LogFeederProps logFeederProps) throws Exception {
+ super.init(logFeederProps);
initParams();
setupSecurity();
createOutgoingBuffer();
@@ -175,8 +175,8 @@ public class OutputSolr extends Output implements CollectionStateWatcher {
}
private void setupSecurity() {
- String jaasFile = LogFeederPropertiesUtil.getSolrJaasFile();
- boolean securityEnabled = LogFeederPropertiesUtil.isSolrKerberosEnabled();
+ String jaasFile = getLogFeederProps().getLogFeederSecurityConfig().getSolrJaasFile();
+ boolean securityEnabled = getLogFeederProps().getLogFeederSecurityConfig().isSolrKerberosEnabled();
if (securityEnabled) {
System.setProperty("java.security.auth.login.config", jaasFile);
HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederPropertiesUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederPropertiesUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederPropertiesUtil.java
deleted file mode 100644
index 1636653..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederPropertiesUtil.java
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.util;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.util.HashMap;
-import java.util.Properties;
-
-import org.apache.ambari.logfeeder.LogFeeder;
-import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Logger;
-
-/**
- * This class contains utility methods used by LogFeeder
- */
-public class LogFeederPropertiesUtil {
- private static final Logger LOG = Logger.getLogger(LogFeederPropertiesUtil.class);
-
- public static final String LOGFEEDER_PROPERTIES_FILE = "logfeeder.properties";
-
- private static Properties props;
- public static Properties getProperties() {
- return props;
- }
-
- public static void loadProperties() throws Exception {
- loadProperties(LOGFEEDER_PROPERTIES_FILE);
- }
-
- /**
- * This method will read the properties from System, followed by propFile and finally from the map
- */
- public static void loadProperties(String propFile) throws Exception {
- LOG.info("Loading properties. propFile=" + propFile);
- props = new Properties(System.getProperties());
- boolean propLoaded = false;
-
- // First get properties file path from environment value
- String propertiesFilePath = System.getProperty("properties");
- if (StringUtils.isNotEmpty(propertiesFilePath)) {
- File propertiesFile = new File(propertiesFilePath);
- if (propertiesFile.exists() && propertiesFile.isFile()) {
- LOG.info("Properties file path set in environment. Loading properties file=" + propertiesFilePath);
- try (FileInputStream fis = new FileInputStream(propertiesFile)) {
- props.load(fis);
- propLoaded = true;
- } catch (Throwable t) {
- LOG.error("Error loading properties file. properties file=" + propertiesFile.getAbsolutePath());
- }
- } else {
- LOG.error("Properties file path set in environment, but file not found. properties file=" + propertiesFilePath);
- }
- }
-
- if (!propLoaded) {
- try (BufferedInputStream bis = (BufferedInputStream) LogFeeder.class.getClassLoader().getResourceAsStream(propFile)) {
- // Properties not yet loaded, let's try from class loader
- if (bis != null) {
- LOG.info("Loading properties file " + propFile + " from classpath");
- props.load(bis);
- propLoaded = true;
- } else {
- LOG.fatal("Properties file not found in classpath. properties file name= " + propFile);
- }
- }
- }
-
- if (!propLoaded) {
- LOG.fatal("Properties file is not loaded.");
- throw new Exception("Properties not loaded");
- }
- }
-
- public static String getStringProperty(String key) {
- return props == null ? null : props.getProperty(key);
- }
-
- public static String getStringProperty(String key, String defaultValue) {
- return props == null ? defaultValue : props.getProperty(key, defaultValue);
- }
-
- public static boolean getBooleanProperty(String key, boolean defaultValue) {
- String value = getStringProperty(key);
- return toBoolean(value, defaultValue);
- }
-
- private static boolean toBoolean(String value, boolean defaultValue) {
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
-
- return "true".equalsIgnoreCase(value) || "yes".equalsIgnoreCase(value);
- }
-
- public static int getIntProperty(String key, int defaultValue) {
- return getIntProperty(key, defaultValue, null, null);
- }
-
- public static int getIntProperty(String key, int defaultValue, Integer minValue, Integer maxValue) {
- String value = getStringProperty(key);
- int retValue = LogFeederUtil.objectToInt(value, defaultValue, ", key=" + key);
- if (minValue != null && retValue < minValue) {
- LOG.info("Minimum rule was applied for " + key + ": " + retValue + " < " + minValue);
- retValue = minValue;
- }
- if (maxValue != null && retValue > maxValue) {
- LOG.info("Maximum rule was applied for " + key + ": " + retValue + " > " + maxValue);
- retValue = maxValue;
- }
- return retValue;
- }
-
- private static final String CLUSTER_NAME_PROPERTY = "cluster.name";
-
- @LogSearchPropertyDescription(
- name = CLUSTER_NAME_PROPERTY,
- description = "The name of the cluster the Log Feeder program runs in.",
- examples = {"cl1"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getClusterName() {
- return getStringProperty(CLUSTER_NAME_PROPERTY);
- }
-
- private static final String TMP_DIR_PROPERTY = "logfeeder.tmp.dir";
- private static final String DEFAULT_TMP_DIR = "/tmp/$username/logfeeder/";
- private static String logFeederTempDir = null;
-
- @LogSearchPropertyDescription(
- name = TMP_DIR_PROPERTY,
- description = "The tmp dir used for creating temporary files.",
- examples = {"/tmp/"},
- defaultValue = DEFAULT_TMP_DIR,
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public synchronized static String getLogFeederTempDir() {
- if (logFeederTempDir == null) {
- String tempDirValue = getStringProperty(TMP_DIR_PROPERTY, DEFAULT_TMP_DIR);
- HashMap<String, String> contextParam = new HashMap<String, String>();
- String username = System.getProperty("user.name");
- contextParam.put("username", username);
- logFeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam);
- }
- return logFeederTempDir;
- }
-
- public static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path";
-
- @LogSearchPropertyDescription(
- name = CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY,
- description = "The jceks file that provides passwords.",
- examples = {"jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder.jceks"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getCredentialStoreProviderPath() {
- return getStringProperty(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY);
- }
-
- private static final String CONFIG_FILES_PROPERTY = "logfeeder.config.files";
-
- @LogSearchPropertyDescription(
- name = CONFIG_FILES_PROPERTY,
- description = "Comma separated list of the config files containing global / output configurations.",
- examples = {"global.json,output.json", "/etc/ambari-logsearch-logfeeder/conf/global.json"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getConfigFiles() {
- return getStringProperty(CONFIG_FILES_PROPERTY);
- }
-
- private static final String CONFIG_DIR_PROPERTY = "logfeeder.config.dir";
-
- @LogSearchPropertyDescription(
- name = CONFIG_DIR_PROPERTY,
- description = "The directory where shipper configuration files are looked for.",
- examples = {"/etc/ambari-logsearch-logfeeder/conf"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getConfigDir() {
- return getStringProperty(CONFIG_DIR_PROPERTY);
- }
-
- public static final String CHECKPOINT_EXTENSION_PROPERTY = "logfeeder.checkpoint.extension";
- public static final String DEFAULT_CHECKPOINT_EXTENSION = ".cp";
-
- @LogSearchPropertyDescription(
- name = CHECKPOINT_EXTENSION_PROPERTY,
- description = "The extension used for checkpoint files.",
- examples = {"ckp"},
- defaultValue = DEFAULT_CHECKPOINT_EXTENSION,
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getCheckPointExtension() {
- return getStringProperty(CHECKPOINT_EXTENSION_PROPERTY, DEFAULT_CHECKPOINT_EXTENSION);
- }
-
- private static final String CHECKPOINT_FOLDER_PROPERTY = "logfeeder.checkpoint.folder";
-
- @LogSearchPropertyDescription(
- name = CHECKPOINT_FOLDER_PROPERTY,
- description = "The folder wher checkpoint files are stored.",
- examples = {"/etc/ambari-logsearch-logfeeder/conf/checkpoints"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getCheckpointFolder() {
- return getStringProperty(CHECKPOINT_FOLDER_PROPERTY);
- }
-
- private static final String CACHE_ENABLED_PROPERTY = "logfeeder.cache.enabled";
- private static final boolean DEFAULT_CACHE_ENABLED = false;
-
- @LogSearchPropertyDescription(
- name = CACHE_ENABLED_PROPERTY,
- description = "Enables the usage of a cache to avoid duplications.",
- examples = {"true"},
- defaultValue = DEFAULT_CACHE_ENABLED + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static boolean isCacheEnabled() {
- return getBooleanProperty(CACHE_ENABLED_PROPERTY, DEFAULT_CACHE_ENABLED);
- }
-
- private static final String CACHE_KEY_FIELD_PROPERTY = "logfeeder.cache.key.field";
- private static final String DEFAULT_CACHE_KEY_FIELD = "log_message";
-
- @LogSearchPropertyDescription(
- name = CACHE_KEY_FIELD_PROPERTY,
- description = "The field which's value should be cached and should be checked for repteitions.",
- examples = {"some_field_prone_to_repeating_value"},
- defaultValue = DEFAULT_CACHE_KEY_FIELD,
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getCacheKeyField() {
- return getStringProperty(CACHE_KEY_FIELD_PROPERTY, DEFAULT_CACHE_KEY_FIELD);
- }
-
- private static final String CACHE_SIZE_PROPERTY = "logfeeder.cache.size";
- private static final int DEFAULT_CACHE_SIZE = 100;
-
- @LogSearchPropertyDescription(
- name = CACHE_SIZE_PROPERTY,
- description = "The number of log entries to cache in order to avoid duplications.",
- examples = {"50"},
- defaultValue = DEFAULT_CACHE_SIZE + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static int getCacheSize() {
- return getIntProperty(CACHE_SIZE_PROPERTY, DEFAULT_CACHE_SIZE);
- }
-
- private static final String CACHE_LAST_DEDUP_ENABLED_PROPERTY = "logfeeder.cache.last.dedup.enabled";
- private static final boolean DEFAULT_CACHE_LAST_DEDUP_ENABLED = false;
-
- @LogSearchPropertyDescription(
- name = CACHE_LAST_DEDUP_ENABLED_PROPERTY,
- description = "Enable filtering directly repeating log entries irrelevant of the time spent between them.",
- examples = {"true"},
- defaultValue = DEFAULT_CACHE_LAST_DEDUP_ENABLED + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static boolean isCacheLastDedupEnabled() {
- return getBooleanProperty(CACHE_LAST_DEDUP_ENABLED_PROPERTY, DEFAULT_CACHE_LAST_DEDUP_ENABLED);
- }
-
- private static final String CACHE_DEDUP_INTERVAL_PROPERTY = "logfeeder.cache.dedup.interval";
- private static final long DEFAULT_CACHE_DEDUP_INTERVAL = 1000;
-
- @LogSearchPropertyDescription(
- name = CACHE_DEDUP_INTERVAL_PROPERTY,
- description = "Maximum number of milliseconds between two identical messages to be filtered out.",
- examples = {"500"},
- defaultValue = DEFAULT_CACHE_DEDUP_INTERVAL + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getCacheDedupInterval() {
- return getStringProperty(CACHE_DEDUP_INTERVAL_PROPERTY, String.valueOf(DEFAULT_CACHE_DEDUP_INTERVAL));
- }
-
- private static final String LOG_FILTER_ENABLE_PROPERTY = "logfeeder.log.filter.enable";
- private static final boolean DEFAULT_LOG_FILTER_ENABLE = false;
-
- @LogSearchPropertyDescription(
- name = LOG_FILTER_ENABLE_PROPERTY,
- description = "Enables the filtering of the log entries by log level filters.",
- examples = {"true"},
- defaultValue = DEFAULT_LOG_FILTER_ENABLE + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static boolean isLogFilterEnabled() {
- return getBooleanProperty(LOG_FILTER_ENABLE_PROPERTY, DEFAULT_LOG_FILTER_ENABLE);
- }
-
- private static final String INCLUDE_DEFAULT_LEVEL_PROPERTY = "logfeeder.include.default.level";
-
- @LogSearchPropertyDescription(
- name = INCLUDE_DEFAULT_LEVEL_PROPERTY,
- description = "Comma separtaed list of the default log levels to be enabled by the filtering.",
- examples = {"FATAL,ERROR,WARN"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getIncludeDefaultLevel() {
- return getStringProperty(INCLUDE_DEFAULT_LEVEL_PROPERTY);
- }
-
- private static final String DEFAULT_SOLR_JAAS_FILE = "/etc/security/keytabs/logsearch_solr.service.keytab";
- private static final String SOLR_JAAS_FILE_PROPERTY = "logfeeder.solr.jaas.file";
-
- @LogSearchPropertyDescription(
- name = SOLR_JAAS_FILE_PROPERTY,
- description = "The jaas file used for solr.",
- examples = {"/etc/ambari-logsearch-logfeeder/conf/logfeeder_jaas.conf"},
- defaultValue = DEFAULT_SOLR_JAAS_FILE,
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getSolrJaasFile() {
- return getStringProperty(SOLR_JAAS_FILE_PROPERTY, DEFAULT_SOLR_JAAS_FILE);
- }
-
- private static final String SOLR_KERBEROS_ENABLE_PROPERTY = "logfeeder.solr.kerberos.enable";
- private static final boolean DEFAULT_SOLR_KERBEROS_ENABLE = false;
-
- @LogSearchPropertyDescription(
- name = SOLR_KERBEROS_ENABLE_PROPERTY,
- description = "Enables using kerberos for accessing solr.",
- examples = {"true"},
- defaultValue = DEFAULT_SOLR_KERBEROS_ENABLE + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static boolean isSolrKerberosEnabled() {
- return getBooleanProperty(SOLR_KERBEROS_ENABLE_PROPERTY, DEFAULT_SOLR_KERBEROS_ENABLE);
- }
-
- private static final String METRICS_COLLECTOR_HOSTS_PROPERTY = "logfeeder.metrics.collector.hosts";
-
- @LogSearchPropertyDescription(
- name = METRICS_COLLECTOR_HOSTS_PROPERTY,
- description = "Comma separtaed list of metric collector hosts.",
- examples = {"c6401.ambari.apache.org"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getMetricsCollectorHosts() {
- return getStringProperty(METRICS_COLLECTOR_HOSTS_PROPERTY);
- }
-
- private static final String METRICS_COLLECTOR_PROTOCOL_PROPERTY = "logfeeder.metrics.collector.protocol";
-
- @LogSearchPropertyDescription(
- name = METRICS_COLLECTOR_PROTOCOL_PROPERTY,
- description = "The protocol used by metric collectors.",
- examples = {"http", "https"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getMetricsCollectorProtocol() {
- return getStringProperty(METRICS_COLLECTOR_PROTOCOL_PROPERTY);
- }
-
- private static final String METRICS_COLLECTOR_PORT_PROPERTY = "logfeeder.metrics.collector.port";
-
- @LogSearchPropertyDescription(
- name = METRICS_COLLECTOR_PORT_PROPERTY,
- description = "The port used by metric collectors.",
- examples = {"6188"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getMetricsCollectorPort() {
- return getStringProperty(METRICS_COLLECTOR_PORT_PROPERTY);
- }
-
- private static final String METRICS_COLLECTOR_PATH_PROPERTY = "logfeeder.metrics.collector.path";
-
- @LogSearchPropertyDescription(
- name = METRICS_COLLECTOR_PATH_PROPERTY,
- description = "The path used by metric collectors.",
- examples = {"/ws/v1/timeline/metrics"},
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getMetricsCollectorPath() {
- return getStringProperty(METRICS_COLLECTOR_PATH_PROPERTY);
- }
-
- private static final String SIMULATE_INPUT_NUMBER_PROPERTY = "logfeeder.simulate.input_number";
- private static final int DEFAULT_SIMULATE_INPUT_NUMBER = 0;
-
- @LogSearchPropertyDescription(
- name = SIMULATE_INPUT_NUMBER_PROPERTY,
- description = "The number of the simulator instances to run with. O means no simulation.",
- examples = {"10"},
- defaultValue = DEFAULT_SIMULATE_INPUT_NUMBER + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static int getSimulateInputNumber() {
- return getIntProperty(SIMULATE_INPUT_NUMBER_PROPERTY, DEFAULT_SIMULATE_INPUT_NUMBER);
- }
-
- private static final String SIMULATE_LOG_LEVEL_PROPERTY = "logfeeder.simulate.log_level";
- private static final String DEFAULT_SIMULATE_LOG_LEVEL = "WARN";
-
- @LogSearchPropertyDescription(
- name = SIMULATE_LOG_LEVEL_PROPERTY,
- description = "The log level to create the simulated log entries with.",
- examples = {"INFO"},
- defaultValue = DEFAULT_SIMULATE_LOG_LEVEL,
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getSimulateLogLevel() {
- return getStringProperty(SIMULATE_LOG_LEVEL_PROPERTY, DEFAULT_SIMULATE_LOG_LEVEL);
- }
-
- private static final String SIMULATE_NUMBER_OF_WORDS_PROPERTY = "logfeeder.simulate.number_of_words";
- private static final int DEFAULT_SIMULATE_NUMBER_OF_WORDS = 1000;
-
- @LogSearchPropertyDescription(
- name = SIMULATE_NUMBER_OF_WORDS_PROPERTY,
- description = "The size of the set of words that may be used to create the simulated log entries with.",
- examples = {"100"},
- defaultValue = DEFAULT_SIMULATE_NUMBER_OF_WORDS + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static int getSimulateNumberOfWords() {
- return getIntProperty(SIMULATE_NUMBER_OF_WORDS_PROPERTY, DEFAULT_SIMULATE_NUMBER_OF_WORDS, 50, 1000000);
- }
-
- private static final String SIMULATE_MIN_LOG_WORDS_PROPERTY = "logfeeder.simulate.min_log_words";
- private static final int DEFAULT_SIMULATE_MIN_LOG_WORDS = 5;
-
- @LogSearchPropertyDescription(
- name = SIMULATE_MIN_LOG_WORDS_PROPERTY,
- description = "The minimum number of words in a simulated log entry.",
- examples = {"3"},
- defaultValue = DEFAULT_SIMULATE_MIN_LOG_WORDS + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static int getSimulateMinLogWords() {
- return getIntProperty(SIMULATE_MIN_LOG_WORDS_PROPERTY, DEFAULT_SIMULATE_MIN_LOG_WORDS, 1, 10);
- }
-
- private static final String SIMULATE_MAX_LOG_WORDS_PROPERTY = "logfeeder.simulate.max_log_words";
- private static final int DEFAULT_SIMULATE_MAX_LOG_WORDS = 5;
-
- @LogSearchPropertyDescription(
- name = SIMULATE_MAX_LOG_WORDS_PROPERTY,
- description = "The maximum number of words in a simulated log entry.",
- examples = {"8"},
- defaultValue = DEFAULT_SIMULATE_MAX_LOG_WORDS + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static int getSimulateMaxLogWords() {
- return getIntProperty(SIMULATE_MAX_LOG_WORDS_PROPERTY, DEFAULT_SIMULATE_MAX_LOG_WORDS, 10, 20);
- }
-
- private static final String SIMULATE_SLEEP_MILLISECONDS_PROPERTY = "logfeeder.simulate.sleep_milliseconds";
- private static final int DEFAULT_SIMULATE_SLEEP_MILLISECONDS = 10000;
-
- @LogSearchPropertyDescription(
- name = SIMULATE_SLEEP_MILLISECONDS_PROPERTY,
- description = "The milliseconds to sleep between creating two simulated log entries.",
- examples = {"5000"},
- defaultValue = DEFAULT_SIMULATE_SLEEP_MILLISECONDS + "",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static int getSimulateSleepMilliseconds() {
- return getIntProperty(SIMULATE_SLEEP_MILLISECONDS_PROPERTY, DEFAULT_SIMULATE_SLEEP_MILLISECONDS);
- }
-
- private static final String SIMULATE_LOG_IDS_PROPERTY = "logfeeder.simulate.log_ids";
-
- @LogSearchPropertyDescription(
- name = SIMULATE_LOG_IDS_PROPERTY,
- description = "The comma separated list of log ids for which to create the simulated log entries.",
- examples = {"ambari_server,zookeeper,infra_solr,logsearch_app"},
- defaultValue = "The log ids of the installed services in the cluster",
- sources = {LOGFEEDER_PROPERTIES_FILE}
- )
- public static String getSimulateLogIds() {
- return getStringProperty(SIMULATE_LOG_IDS_PROPERTY);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
deleted file mode 100644
index 6bcaac9..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.util;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
-
-import java.io.File;
-import java.nio.charset.Charset;
-
-public class SSLUtil {
- private static final Logger LOG = Logger.getLogger(SSLUtil.class);
-
- private static final String KEYSTORE_LOCATION_ARG = "javax.net.ssl.keyStore";
- private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore";
- private static final String KEYSTORE_TYPE_ARG = "javax.net.ssl.keyStoreType";
- private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType";
- private static final String KEYSTORE_PASSWORD_ARG = "javax.net.ssl.keyStorePassword";
- private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword";
- private static final String KEYSTORE_PASSWORD_PROPERTY_NAME = "logfeeder_keystore_password";
- private static final String TRUSTSTORE_PASSWORD_PROPERTY_NAME = "logfeeder_truststore_password";
- private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt";
- private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt";
-
- private static final String LOGFEEDER_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys";
- private static final String LOGFEEDER_STORE_DEFAULT_PASSWORD = "bigdata";
-
- private SSLUtil() {
- throw new UnsupportedOperationException();
- }
-
- public static String getKeyStoreLocation() {
- return System.getProperty(KEYSTORE_LOCATION_ARG);
- }
-
- public static String getKeyStoreType() {
- return System.getProperty(KEYSTORE_TYPE_ARG);
- }
-
- public static String getKeyStorePassword() {
- return System.getProperty(KEYSTORE_PASSWORD_ARG);
- }
-
- public static String getTrustStoreLocation() {
- return System.getProperty(TRUSTSTORE_LOCATION_ARG);
- }
-
- public static String getTrustStoreType() {
- return System.getProperty(TRUSTSTORE_TYPE_ARG);
- }
-
- public static String getTrustStorePassword() {
- return System.getProperty(TRUSTSTORE_PASSWORD_ARG);
- }
-
- public static void ensureStorePasswords() {
- ensureStorePassword(KEYSTORE_LOCATION_ARG, KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_PROPERTY_NAME, KEYSTORE_PASSWORD_FILE);
- ensureStorePassword(TRUSTSTORE_LOCATION_ARG, TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_PROPERTY_NAME, TRUSTSTORE_PASSWORD_FILE);
- }
-
- private static void ensureStorePassword(String locationArg, String pwdArg, String propertyName, String fileName) {
- if (StringUtils.isNotEmpty(System.getProperty(locationArg)) && StringUtils.isEmpty(System.getProperty(pwdArg))) {
- String password = getPassword(propertyName, fileName);
- System.setProperty(pwdArg, password);
- }
- }
-
- private static String getPassword(String propertyName, String fileName) {
- String credentialStorePassword = getPasswordFromCredentialStore(propertyName);
- if (credentialStorePassword != null) {
- return credentialStorePassword;
- }
-
- String filePassword = getPasswordFromFile(fileName);
- if (filePassword != null) {
- return filePassword;
- }
-
- return LOGFEEDER_STORE_DEFAULT_PASSWORD;
- }
-
- private static String getPasswordFromCredentialStore(String propertyName) {
- try {
- String providerPath = LogFeederPropertiesUtil.getCredentialStoreProviderPath();
- if (providerPath == null) {
- return null;
- }
-
- Configuration config = new Configuration();
- config.set(LogFeederPropertiesUtil.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, providerPath);
- char[] passwordChars = config.getPassword(propertyName);
- return (ArrayUtils.isNotEmpty(passwordChars)) ? new String(passwordChars) : null;
- } catch (Exception e) {
- LOG.warn(String.format("Could not load password %s from credential store, using default password", propertyName));
- return null;
- }
- }
-
- private static String getPasswordFromFile(String fileName) {
- try {
- File pwdFile = new File(LOGFEEDER_CERT_DEFAULT_FOLDER, fileName);
- if (!pwdFile.exists()) {
- FileUtils.writeStringToFile(pwdFile, LOGFEEDER_STORE_DEFAULT_PASSWORD, Charset.defaultCharset());
- return LOGFEEDER_STORE_DEFAULT_PASSWORD;
- } else {
- return FileUtils.readFileToString(pwdFile, Charset.defaultCharset());
- }
- } catch (Exception e) {
- LOG.warn("Exception occurred during read/write password file for keystore/truststore.", e);
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
index 4ab2eb2..e54c974 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
@@ -22,7 +22,7 @@
"post_map_values": {
"logtime": {
"map_date": {
- "date_pattern": "yyyy-MM-dd HH:mm:ss,SSS"
+ "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS"
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
index eb20665..d01160c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
@@ -23,7 +23,6 @@
<param name="Target" value="System.out" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
- <!-- <param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/> -->
</layout>
</appender>
@@ -43,10 +42,8 @@
<param name="maxFileSize" value="10MB" />
<param name="maxBackupIndex" value="10" />
<layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" />
- </appender>
-
+ </appender>
- <!-- Logs to suppress BEGIN -->
<category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
<priority value="error" />
<appender-ref ref="daily_rolling_file" />
@@ -56,19 +53,15 @@
<priority value="fatal" />
<appender-ref ref="daily_rolling_file" />
</category>
- <!-- Logs to suppress END -->
<category name="org.apache.ambari.logfeeder" additivity="false">
<priority value="INFO" />
- <appender-ref ref="console" />
- <!-- <appender-ref ref="daily_rolling_file" /> -->
- <appender-ref ref="rolling_file_json"/>
+ <appender-ref ref="console" />
</category>
<root>
- <priority value="warn" />
- <!-- <appender-ref ref="console" /> -->
- <!-- <appender-ref ref="daily_rolling_file" /> -->
+ <priority value="info"/>
+ <appender-ref ref="console"/>
</root>
</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 115778b..4b49446 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -14,23 +14,27 @@
# limitations under the License.
cluster.name=cl1
-logfeeder.checkpoint.folder=target/checkpoints
+logsearch.config.zk_connect_string=localhost:2181
+
logfeeder.metrics.collector.hosts=
-logfeeder.config.dir=target/classes/log-samples/shipper-conf/
-logfeeder.config.files=target/classes/log-samples/shipper-conf/global.config.json,\
- target/classes/log-samples/shipper-conf/input.config-sample.json,\
- target/classes/log-samples/shipper-conf/output.config-sample.json
+logfeeder.checkpoint.folder=${LOGFEEDER_RELATIVE_LOCATION:}target/checkpoints
+logfeeder.config.dir=${LOGFEEDER_RELATIVE_LOCATION:}target/classes/log-samples/shipper-conf/
+logfeeder.config.files=${LOGFEEDER_RELATIVE_LOCATION:}target/classes/log-samples/shipper-conf/global.config.json,\
+ ${LOGFEEDER_RELATIVE_LOCATION:}target/classes/log-samples/shipper-conf/output.config-sample.json
+
logfeeder.log.filter.enable=true
+
logfeeder.solr.config.interval=5
logfeeder.solr.core.config.name=history
logfeeder.solr.zk_connect_string=localhost:2181
+
logfeeder.cache.enabled=true
logfeeder.cache.size=100
logfeeder.cache.key.field=log_message
logfeeder.cache.dedup.interval=1000
logfeeder.cache.last.dedup.enabled=true
-logsearch.config.zk_connect_string=localhost:2181
+
logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
#logfeeder tmp dir
-logfeeder.tmp.dir=target/tmp
+logfeeder.tmp.dir=${LOGFEEDER_RELATIVE_LOCATION:}target/tmp
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
index 8d7e86c..e3a822a 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
@@ -20,6 +20,7 @@ package org.apache.ambari.logfeeder.filter;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.OutputManager;
@@ -52,7 +53,7 @@ public class FilterGrokTest {
filterGrok.loadConfig(filterGrokDescriptor);
filterGrok.setOutputManager(mockOutputManager);
filterGrok.setInput(EasyMock.mock(Input.class));
- filterGrok.init();
+ filterGrok.init(new LogFeederProps());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index acc3d4d..ef10c46 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -26,6 +26,7 @@ import java.util.TimeZone;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.common.LogFeederException;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl;
@@ -54,7 +55,7 @@ public class FilterJSONTest {
filterJson = new FilterJSON();
filterJson.loadConfig(filterJsonDescriptor);
filterJson.setOutputManager(mockOutputManager);
- filterJson.init();
+ filterJson.init(new LogFeederProps());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
index ae978fb..4a85b88 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
@@ -20,6 +20,7 @@ package org.apache.ambari.logfeeder.filter;
import java.util.Map;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterKeyValueDescriptorImpl;
@@ -49,7 +50,7 @@ public class FilterKeyValueTest {
filterKeyValue = new FilterKeyValue();
filterKeyValue.loadConfig(filterKeyValueDescriptor);
filterKeyValue.setOutputManager(mockOutputManager);
- filterKeyValue.init();
+ filterKeyValue.init(new LogFeederProps());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
index efebc08..01b4e54 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
@@ -24,6 +24,8 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
+import org.apache.ambari.logfeeder.conf.LogEntryCacheConfig;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputFileDescriptorImpl;
@@ -61,6 +63,8 @@ public class InputFileTest {
private InputMarker testInputMarker;
+ private LogFeederProps logFeederProps;
+
@Rule
public ExpectedException expectedException = ExpectedException.none();
@@ -74,6 +78,9 @@ public class InputFileTest {
@Before
public void setUp() throws Exception {
+ logFeederProps = new LogFeederProps();
+ LogEntryCacheConfig logEntryCacheConfig = new LogEntryCacheConfig();
+ logFeederProps.setLogEntryCacheConfig(logEntryCacheConfig);
}
public void init(String path) throws Exception {
@@ -87,7 +94,7 @@ public class InputFileTest {
Filter capture = new Filter() {
@Override
- public void init() {
+ public void init(LogFeederProps logFeederProps) {
}
@Override
@@ -103,7 +110,7 @@ public class InputFileTest {
inputFile = new InputFile();
inputFile.loadConfig(inputFileDescriptor);
inputFile.addFilter(capture);
- inputFile.init();
+ inputFile.init(logFeederProps);
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/31e8e55a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
index 46fbc3b..9dba349 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.List;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.junit.Test;
@@ -62,10 +63,12 @@ public class InputManagerTest {
Input input1 = strictMock(Input.class);
Input input2 = strictMock(Input.class);
Input input3 = strictMock(Input.class);
-
- input1.init(); expectLastCall();
- input2.init(); expectLastCall();
- input3.init(); expectLastCall();
+
+ LogFeederProps logFeederProps = new LogFeederProps();
+
+ input1.init(logFeederProps); expectLastCall();
+ input2.init(logFeederProps); expectLastCall();
+ input3.init(logFeederProps); expectLastCall();
expect(input1.isReady()).andReturn(true);
expect(input2.isReady()).andReturn(true);
@@ -78,6 +81,7 @@ public class InputManagerTest {
replay(input1, input2, input3);
InputManager manager = new InputManager();
+ manager.setLogFeederProps(logFeederProps);
manager.add("serviceName", input1);
manager.add("serviceName", input2);
manager.add("serviceName", input3);
[21/51] [abbrv] ambari git commit: AMBARI-22670 Ambari 3.0: Implement
new design for Admin View: Integrate visual-search box. (atkach)
Posted by am...@apache.org.
AMBARI-22670 Ambari 3.0: Implement new design for Admin View: Integrate visual-search box. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/30d89a33
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/30d89a33
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/30d89a33
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 30d89a3306dddf24fb2206ff22bc85c40458debd
Parents: 90554f3
Author: Andrii Tkach <at...@apache.org>
Authored: Tue Dec 19 14:53:08 2017 +0200
Committer: Andrii Tkach <at...@apache.org>
Committed: Tue Dec 19 14:53:08 2017 +0200
----------------------------------------------------------------------
.../main/resources/ui/admin-web/app/index.html | 1 +
.../controllers/ambariViews/ViewsListCtrl.js | 104 +++--
.../remoteClusters/RemoteClustersListCtrl.js | 3 +-
.../stackVersions/StackVersionsListCtrl.js | 1 +
.../app/scripts/directives/comboSearch.js | 455 +++++++++++++++++++
.../ui/admin-web/app/styles/combo-search.css | 164 +++++++
.../resources/ui/admin-web/app/styles/main.css | 36 ++
.../app/views/ambariViews/viewsList.html | 45 +-
.../app/views/directives/comboSearch.html | 63 +++
.../app/views/remoteClusters/list.html | 2 +-
.../admin-web/app/views/stackVersions/list.html | 2 +-
.../ambariViews/ViewsListCtrl_test.js | 167 +++++++
.../test/unit/directives/comboSearch_test.js | 242 ++++++++++
13 files changed, 1201 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/index.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/index.html b/ambari-admin/src/main/resources/ui/admin-web/app/index.html
index bf033e6..a1346ed 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/index.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/index.html
@@ -150,6 +150,7 @@
<script src="scripts/directives/PasswordVerify.js"></script>
<script src="scripts/directives/disabledTooltip.js"></script>
<script src="scripts/directives/editableList.js"></script>
+<script src="scripts/directives/comboSearch.js"></script>
<script src="scripts/services/Utility.js"></script>
<script src="scripts/services/UserConstants.js"></script>
<script src="scripts/services/User.js"></script>
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
index 8b37dca..8c61a25 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
@@ -24,6 +24,29 @@ angular.module('ambariAdminConsole')
$scope.isLoading = false;
$scope.minInstanceForPagination = Settings.minRowsToShowPagination;
+ $scope.filters = [
+ {
+ key: 'short_url_name',
+ label: $t('common.name'),
+ options: []
+ },
+ {
+ key: 'url',
+ label: $t('urls.url'),
+ options: []
+ },
+ {
+ key: 'view_name',
+ label: $t('views.table.viewType'),
+ options: []
+ },
+ {
+ key: 'instance_name',
+ label: $t('urls.viewInstance'),
+ options: []
+ }
+ ];
+
function checkViewVersionStatus(view, versionObj, versionNumber) {
var deferred = View.checkViewVersionStatus(view.view_name, versionNumber);
@@ -66,27 +89,13 @@ angular.module('ambariAdminConsole')
$scope.instances.push(instance.ViewInstanceInfo);
});
});
- initTypeFilter();
+ $scope.initFilterOptions();
$scope.filterInstances();
}).catch(function (data) {
Alert.error($t('views.alerts.cannotLoadViews'), data.data.message);
});
}
- function initTypeFilter() {
- var uniqTypes = $.unique($scope.instances.map(function(instance) {
- return instance.view_name;
- }));
- $scope.typeFilterOptions = [ { label: $t('common.all'), value: '*'} ]
- .concat(uniqTypes.map(function(type) {
- return {
- label: type,
- value: type
- };
- }));
- $scope.instanceTypeFilter = $scope.typeFilterOptions[0];
- }
-
function showInstancesOnPage() {
var startIndex = ($scope.currentPage - 1) * $scope.instancesPerPage + 1;
var endIndex = $scope.currentPage * $scope.instancesPerPage;
@@ -110,11 +119,7 @@ angular.module('ambariAdminConsole')
$scope.instances = [];
$scope.instancesPerPage = 10;
$scope.currentPage = 1;
- $scope.instanceNameFilter = '';
- $scope.instanceUrlFilter = '';
$scope.maxVisiblePages = 10;
- $scope.isNotEmptyFilter = true;
- $scope.instanceTypeFilter = '';
$scope.tableInfo = {
filtered: 0,
showed: 0
@@ -122,25 +127,46 @@ angular.module('ambariAdminConsole')
loadViews();
- $scope.filterInstances = function() {
+ $scope.initFilterOptions = function() {
+ $scope.filters.forEach(function(filter) {
+ filter.options = $.unique($scope.instances.map(function(instance) {
+ if (filter.key === 'url') {
+ return '/main/view/' + instance.view_name + '/' + instance.short_url;
+ }
+ return instance[filter.key];
+ })).map(function(item) {
+ return {
+ key: item,
+ label: item
+ }
+ });
+ });
+ };
+
+ $scope.filterInstances = function(appliedFilters) {
var filteredCount = 0;
angular.forEach($scope.instances, function(instance) {
- if ($scope.instanceNameFilter && instance.short_url_name.indexOf($scope.instanceNameFilter) === -1) {
- return instance.isFiltered = false;
- }
- if ($scope.instanceUrlFilter && ('/main/view/'+ instance.view_name + '/' + instance.short_url).indexOf($scope.instanceUrlFilter) === -1) {
- return instance.isFiltered = false;
- }
- if ($scope.instanceTypeFilter.value !== '*' && instance.view_name.indexOf($scope.instanceTypeFilter.value) === -1) {
- return instance.isFiltered = false;
- }
- filteredCount++;
- instance.isFiltered = true;
+ instance.isFiltered = !(appliedFilters && appliedFilters.length > 0 && appliedFilters.some(function(filter) {
+ if (filter.key === 'url') {
+ return filter.values.every(function(value) {
+ return ('/main/view/' + instance.view_name + '/' + instance.short_url).indexOf(value) === -1;
+ });
+ }
+ return filter.values.every(function(value) {
+ return instance[filter.key].indexOf(value) === -1;
+ });
+ }));
+
+ filteredCount += ~~instance.isFiltered;
});
$scope.tableInfo.filtered = filteredCount;
$scope.resetPagination();
};
+ $scope.toggleSearchBox = function() {
+ $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
+ };
+
$scope.pageChanged = function() {
showInstancesOnPage();
};
@@ -150,22 +176,6 @@ angular.module('ambariAdminConsole')
showInstancesOnPage();
};
- $scope.clearFilters = function () {
- $scope.instanceNameFilter = '';
- $scope.instanceUrlFilter = '';
- $scope.instanceTypeFilter = $scope.typeFilterOptions[0];
- $scope.resetPagination();
- };
-
- $scope.$watch(
- function (scope) {
- return Boolean(scope.instanceNameFilter || scope.instanceUrlFilter || (scope.instanceTypeFilter && scope.instanceTypeFilter.value !== '*'));
- },
- function (newValue, oldValue, scope) {
- scope.isNotEmptyFilter = newValue;
- }
- );
-
$scope.cloneInstance = function(instanceClone) {
$scope.createInstance(instanceClone);
};
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
index 9d47307..4726357 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
@@ -18,9 +18,10 @@
'use strict';
angular.module('ambariAdminConsole')
-.controller('RemoteClustersListCtrl', ['$scope', '$routeParams', '$translate', 'RemoteCluster', function ($scope, $routeParams, $translate, RemoteCluster) {
+.controller('RemoteClustersListCtrl', ['$scope', '$routeParams', '$translate', 'RemoteCluster', 'Settings', function ($scope, $routeParams, $translate, RemoteCluster, Settings) {
var $t = $translate.instant;
+ $scope.minInstanceForPagination = Settings.minRowsToShowPagination;
$scope.clusterName = $routeParams.clusterName;
$scope.isLoading = false;
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
index 003d472..ae00978 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
@@ -23,6 +23,7 @@ angular.module('ambariAdminConsole')
$scope.getConstant = function (key) {
return $t(key).toLowerCase();
};
+ $scope.minInstanceForPagination = Settings.minRowsToShowPagination;
$scope.isLoading = false;
$scope.clusterName = $routeParams.clusterName;
$scope.filter = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
new file mode 100644
index 0000000..af25167
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
@@ -0,0 +1,455 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+angular.module('ambariAdminConsole')
+.directive('comboSearch', function() {
+ return {
+ restrict: 'E',
+ templateUrl: 'views/directives/comboSearch.html',
+ scope: {
+ suggestions: '=',
+ filterChange: '=',
+ placeholder: '@',
+ supportCategories: '@'
+ },
+ controller: ['$scope', function($scope) {
+ return {
+ suggestions: $scope.suggestions,
+ placeholder: $scope.placeholder,
+ filterChange: $scope.filterChange,
+ supportCategories: $scope.supportCategories === "true"
+ }
+ }],
+ link: function($scope, $elem, $attr, $ctrl) {
+ var idCounter = 1;
+ var suggestions = $ctrl.suggestions;
+ var supportCategories = $ctrl.supportCategories;
+ var mainInputElement = $elem.find('.main-input.combo-search-input');
+ $scope.paceholder = $ctrl.placeholder;
+ $scope.searchFilterInput = '';
+ $scope.filterSuggestions = [];
+ $scope.showAutoComplete = false;
+ $scope.appliedFilters = [];
+
+ attachInputWidthSetter(mainInputElement);
+ initKeyHandlers();
+ initBlurHandler();
+
+ $scope.$watch(function () {
+ return $scope.appliedFilters.length;
+ }, function () {
+ attachInputWidthSetter($elem.find('.combo-search-input'));
+ });
+
+ $scope.removeFilter = function(filter) {
+ $scope.appliedFilters = $scope.appliedFilters.filter(function(item) {
+ return filter.id !== item.id;
+ });
+ $scope.observeSearchFilterInput(event);
+ mainInputElement.focus();
+ $scope.updateFilters($scope.appliedFilters);
+ };
+
+ $scope.clearFilters = function() {
+ $scope.appliedFilters = [];
+ $scope.updateFilters($scope.appliedFilters);
+ };
+
+ $scope.selectFilter = function(filter, event) {
+ var newAppliedFilter = {
+ id: 'filter_' + idCounter++,
+ currentOption: null,
+ filteredOptions: [],
+ searchOptionInput: '',
+ key: filter.key,
+ label: filter.label,
+ options: filter.options || [],
+ showAutoComplete: false
+ };
+ $scope.appliedFilters.push(newAppliedFilter);
+ if (event) {
+ event.stopPropagation();
+ event.preventDefault();
+ }
+ $scope.isEditing = false;
+ $scope.showAutoComplete = false;
+ $scope.searchFilterInput = '';
+ _.debounce(function() {
+ $('input[name=' + newAppliedFilter.id + ']').focus().width(4);
+ }, 100)();
+ };
+
+ $scope.selectOption = function(event, option, filter) {
+ $('input[name=' + filter.id + ']').val(option.label).trigger('input');
+ filter.showAutoComplete = false;
+ mainInputElement.focus();
+ $scope.observeSearchFilterInput(event);
+ filter.currentOption = option;
+ $scope.updateFilters($scope.appliedFilters);
+ };
+
+ $scope.hideAutocomplete = function(filter) {
+ _.debounce(function() {
+ if (filter) {
+ filter.showAutoComplete = false;
+ } else {
+ if (!$scope.isEditing) {
+ $scope.showAutoComplete = false;
+ }
+ }
+ $scope.$apply();
+ }, 100)();
+ };
+
+ $scope.forceFocus = function(event, filter) {
+ $(event.currentTarget).find('.combo-search-input').focus();
+ $scope.showAutoComplete = false;
+ $scope.observeSearchOptionInput(filter);
+ event.stopPropagation();
+ event.preventDefault();
+ };
+
+ $scope.makeActive = function(active, all) {
+ if (active.isCategory) {
+ return false;
+ }
+ all.forEach(function(item) {
+ item.active = active.key === item.key;
+ });
+ };
+
+ $scope.observeSearchFilterInput = function(event) {
+ if (event) {
+ mainInputElement.focus();
+ $scope.isEditing = true;
+ event.stopPropagation();
+ event.preventDefault();
+ }
+
+ var filteredSuggestions = suggestions.filter(function(item) {
+ return (!$scope.searchFilterInput || item.label.toLowerCase().indexOf($scope.searchFilterInput.toLowerCase()) !== -1);
+ });
+ if (filteredSuggestions.length > 0) {
+ $scope.makeActive(filteredSuggestions[0], filteredSuggestions);
+ $scope.showAutoComplete = true;
+ } else {
+ $scope.showAutoComplete = false;
+ }
+ $scope.filterSuggestions = supportCategories ? formatCategorySuggestions(filteredSuggestions) : filteredSuggestions;
+ };
+
+ $scope.observeSearchOptionInput = function(filter) {
+ var appliedOptions = {};
+ $scope.appliedFilters.forEach(function(item) {
+ if (item.key === filter.key && item.currentOption) {
+ appliedOptions[item.currentOption.key] = true;
+ }
+ });
+
+ if (filter.currentOption && filter.currentOption.key !== filter.searchOptionInput) {
+ filter.currentOption = null;
+ }
+ filter.filteredOptions = filter.options.filter(function(option) {
+ return !(option.key === '' || option.key === undefined || appliedOptions[option.key])
+ && (!filter.searchOptionInput || option.label.toLowerCase().indexOf(filter.searchOptionInput.toLowerCase()) !== -1);
+ });
+ filter.showAutoComplete = filter.filteredOptions.length > 0;
+ if (filter.filteredOptions.length > 0) {
+ $scope.makeActive(filter.filteredOptions[0], filter.filteredOptions);
+ }
+ };
+
+ $scope.extractFilters = function(filters) {
+ var map = {};
+ var result = [];
+
+ filters.forEach(function(filter) {
+ if (filter.currentOption) {
+ if (!map[filter.key]) {
+ map[filter.key] = [];
+ }
+ map[filter.key].push(filter.currentOption.key);
+ }
+ });
+ for(var key in map) {
+ result.push({
+ key: key,
+ values: map[key]
+ });
+ }
+ return result;
+ };
+
+ $scope.updateFilters = function(appliedFilters) {
+ $ctrl.filterChange($scope.extractFilters(appliedFilters));
+ };
+
+ function formatCategorySuggestions(suggestions) {
+ var categories = {};
+ var result = [];
+ suggestions.forEach(function(item) {
+ if (!item.category) {
+ item.category = 'default';
+ }
+ if (!categories[item.category]) {
+ categories[item.category] = [];
+ }
+ categories[item.category].push(item);
+ });
+
+ for(var cat in categories) {
+ result.push({
+ key: cat,
+ label: cat,
+ isCategory: true,
+ isDefault: cat === 'default'
+ });
+ result = result.concat(categories[cat]);
+ }
+ return result;
+ }
+
+ function initBlurHandler() {
+ $(document).click(function() {
+ $scope.isEditing = false;
+ $scope.hideAutocomplete();
+ });
+ }
+
+ function findActiveByName(array, name) {
+ for (var i = 0; i < array.length; i++) {
+ if (array[i].id === name) {
+ return i;
+ }
+ }
+ return null;
+ }
+
+ function findActiveByProperty(array) {
+ for (var i = 0; i < array.length; i++) {
+ if (array[i].active) {
+ return i;
+ }
+ }
+ return 0;
+ }
+
+ function focusInput(filter) {
+ $('input[name=' + filter.id + ']').focus();
+ $scope.showAutoComplete = false;
+ $scope.observeSearchOptionInput(filter);
+ }
+
+ function initKeyHandlers() {
+ $(document).keydown(function(event) {
+ if (event.which === 13) { // "Enter" key
+ enterKeyHandler();
+ $scope.$apply();
+ }
+ if (event.which === 8) { // "Backspace" key
+ backspaceKeyHandler(event);
+ $scope.$apply();
+ }
+ if (event.which === 38) { // "Up" key
+ upKeyHandler();
+ $scope.$apply();
+ }
+ if (event.which === 40) { // "Down" key
+ downKeyHandler();
+ $scope.$apply();
+ }
+ if (event.which === 39) { // "Right Arrow" key
+ rightArrowKeyHandler();
+ $scope.$apply();
+ }
+ if (event.which === 37) { // "Left Arrow" key
+ leftArrowKeyHandler();
+ $scope.$apply();
+ }
+ });
+ }
+
+ function leftArrowKeyHandler() {
+ var activeElement = $(document.activeElement);
+ if (activeElement.is('input') && activeElement[0].selectionStart === 0) {
+ if (activeElement.hasClass('main-input')) {
+ focusInput($scope.appliedFilters[$scope.appliedFilters.length - 1]);
+ } else {
+ var activeIndex = findActiveByName($scope.appliedFilters, activeElement.attr('name'));
+ if (activeIndex !== null && activeIndex > 0) {
+ focusInput($scope.appliedFilters[activeIndex - 1]);
+ }
+ }
+ }
+ }
+
+ function rightArrowKeyHandler() {
+ var activeElement = $(document.activeElement);
+ if (activeElement.is('input') && activeElement[0].selectionStart === activeElement.val().length) {
+ if (!activeElement.hasClass('main-input')) {
+ var activeIndex = findActiveByName($scope.appliedFilters, activeElement.attr('name'));
+ if (activeIndex !== null) {
+ if (activeIndex === $scope.appliedFilters.length - 1) {
+ mainInputElement.focus();
+ $scope.observeSearchFilterInput();
+ } else {
+ focusInput($scope.appliedFilters[activeIndex + 1]);
+ }
+ }
+ }
+ }
+ }
+
+ function downKeyHandler() {
+ var activeIndex = 0;
+ var nextIndex = null;
+
+ if ($scope.showAutoComplete) {
+ activeIndex = findActiveByProperty($scope.filterSuggestions);
+ if (activeIndex < $scope.filterSuggestions.length - 1) {
+ if ($scope.filterSuggestions[activeIndex + 1].isCategory && activeIndex + 2 < $scope.filterSuggestions.length) {
+ nextIndex = activeIndex + 2;
+ } else {
+ nextIndex = activeIndex + 1;
+ }
+ } else {
+ nextIndex = ($scope.filterSuggestions[0].isCategory) ? 1 : 0;
+ }
+ if (nextIndex !== null) {
+ $scope.makeActive($scope.filterSuggestions[nextIndex], $scope.filterSuggestions);
+ }
+ } else {
+ var activeAppliedFilters = $scope.appliedFilters.filter(function(item) {
+ return item.showAutoComplete;
+ });
+ if (activeAppliedFilters.length > 0) {
+ var filteredOptions = activeAppliedFilters[0].filteredOptions;
+ activeIndex = findActiveByProperty(filteredOptions);
+ nextIndex = (activeIndex < filteredOptions.length - 1) ? activeIndex + 1 : 0;
+ }
+ if (nextIndex !== null) {
+ $scope.makeActive(filteredOptions[nextIndex], filteredOptions);
+ }
+ }
+ }
+
+ function upKeyHandler() {
+ var activeIndex = 0;
+ var nextIndex = null;
+
+ if ($scope.showAutoComplete) {
+ activeIndex = findActiveByProperty($scope.filterSuggestions);
+ if (activeIndex > 0) {
+ if ($scope.filterSuggestions[activeIndex - 1].isCategory) {
+ nextIndex = (activeIndex - 2 > 0) ? activeIndex - 2 : $scope.filterSuggestions.length - 1;
+ } else {
+ nextIndex = activeIndex - 1;
+ }
+ } else {
+ nextIndex = $scope.filterSuggestions.length - 1;
+ }
+ if (nextIndex !== null) {
+ $scope.makeActive($scope.filterSuggestions[nextIndex], $scope.filterSuggestions);
+ }
+ } else {
+ var activeAppliedFilters = $scope.appliedFilters.filter(function(item) {
+ return item.showAutoComplete;
+ });
+ if (activeAppliedFilters.length > 0) {
+ var filteredOptions = activeAppliedFilters[0].filteredOptions;
+ activeIndex = findActiveByProperty(filteredOptions);
+ nextIndex = (activeIndex > 0) ? activeIndex - 1 : filteredOptions.length - 1;
+ }
+ if (nextIndex !== null) {
+ $scope.makeActive(filteredOptions[nextIndex], filteredOptions);
+ }
+ }
+ }
+
+ function enterKeyHandler() {
+ if ($scope.showAutoComplete) {
+ var activeFilters = $scope.filterSuggestions.filter(function(item) {
+ return item.active;
+ });
+ if (activeFilters.length > 0) {
+ $scope.selectFilter(activeFilters[0]);
+ }
+ } else {
+ var activeAppliedFilters = $scope.appliedFilters.filter(function(item) {
+ return item.showAutoComplete;
+ });
+ if (activeAppliedFilters.length > 0) {
+ var activeOptions = activeAppliedFilters[0].filteredOptions.filter(function(item) {
+ return item.active;
+ });
+ if (activeOptions.length > 0) {
+ $scope.selectOption(null, activeOptions[0], activeAppliedFilters[0]);
+ }
+ } else {
+ $scope.appliedFilters.filter(function(item) {
+ return !item.currentOption;
+ }).forEach(function(item) {
+ if (item.searchOptionInput !== '') {
+ $scope.selectOption(null, {
+ key: item.searchOptionInput,
+ label: item.searchOptionInput
+ }, item);
+ }
+ });
+ }
+ }
+ }
+
+ function backspaceKeyHandler (event) {
+ if ($(document.activeElement).is('input') && $(document.activeElement)[0].selectionStart === 0) {
+ if ($(document.activeElement).hasClass('main-input') && $scope.appliedFilters.length > 0) {
+ var lastFilter = $scope.appliedFilters[$scope.appliedFilters.length - 1];
+ focusInput(lastFilter);
+ event.stopPropagation();
+ event.preventDefault();
+ } else {
+ var name = $(document.activeElement).attr('name');
+ var activeFilter = $scope.appliedFilters.filter(function(item) {
+ return name === item.id;
+ })[0];
+ if (activeFilter) {
+ $scope.removeFilter(activeFilter);
+ }
+ }
+ }
+ }
+
+ function attachInputWidthSetter(element) {
+ var textPadding = 4;
+ element.on('input', function() {
+ var inputWidth = $(this).textWidth();
+ $(this).css({
+ width: inputWidth + textPadding
+ })
+ }).trigger('input');
+ }
+ }
+ };
+});
+
+$.fn.textWidth = function(text, font) {
+ if (!$.fn.textWidth.fakeEl) $.fn.textWidth.fakeEl = $('<span>').hide().appendTo(document.body);
+ $.fn.textWidth.fakeEl.text(text || this.val() || this.text() || this.attr('placeholder')).css('font', font || this.css('font'));
+ return $.fn.textWidth.fakeEl.width();
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/styles/combo-search.css
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/styles/combo-search.css b/ambari-admin/src/main/resources/ui/admin-web/app/styles/combo-search.css
new file mode 100644
index 0000000..ee9909c
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/styles/combo-search.css
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.combo-search .combo-search-inner {
+ position: relative;
+ border: 1px solid #ccc;
+ border-radius: 4px;
+ box-shadow: 0 0 1px #fff inset;
+ min-height: 34px;
+ line-height: 1em;
+ cursor: text;
+ padding-right: 30px;
+}
+
+.combo-search .combo-search-close {
+ font-size: 13px;
+ color: #999;
+ position: absolute;
+ right: 10px;
+ top: 30%;
+ cursor: pointer;
+}
+.combo-search .combo-search-close:hover {
+ color: #333;
+}
+.combo-search .combo-search-input {
+ background: transparent;
+ display: inline-block;
+ min-width: 4px;
+ width: 4px;
+ line-height: 10px;
+ height: 100%;
+ border: none;
+ outline: none;
+ margin-left: 1px;
+}
+
+.combo-search .combo-search-input-wrapper {
+ display: inline-block;
+ position: relative;
+ height: 32px;
+ margin-left: 5px;
+}
+
+.combo-search .combo-search-content {
+ display: inline-block;
+}
+
+.combo-search .combo-search-dropdown {
+ position: absolute;
+ border: 1px solid #ccc;
+ border-radius: 4px;
+ background-color: #fff;
+ cursor: pointer;
+ z-index: 10;
+ padding: 0;
+ margin: 0;
+ width: auto;
+ min-width: 80px;
+ max-width: 220px;
+ max-height: 240px;
+ overflow-y: auto;
+ overflow-x: hidden;
+ font-size: 13px;
+ top: 30px;
+ left: 5px;
+ box-shadow: 3px 4px 5px -2px rgba(0, 0, 0, 0.5);
+}
+
+.combo-search .combo-search-dropdown ul {
+ max-height: 250px;
+ list-style: none;
+ padding: 0;
+ margin: 0;
+}
+
+.combo-search .filter a {
+ display: block;
+ width: auto;
+ text-decoration: none;
+ color: initial;
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ background: none;
+ border: none;
+ padding: 3px 10px 5px 5px;
+}
+
+.combo-search .category a {
+ display: block;
+ width: auto;
+ text-decoration: none;
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ padding: 3px 10px 5px 5px;
+ text-transform: capitalize;
+ font-size: 11px;
+ font-weight: bold;
+ color: white;
+ cursor: default;
+ border-bottom: 1px solid #A2A2A2;
+ background-color: #B7B7B7;
+ text-shadow: 0 -1px 0 #999;
+}
+
+.combo-search .filter a.active {
+ background-color: #ddd;
+}
+
+.combo-search .combo-search-applied-filter {
+ position: relative;
+ display: inline-block;
+ padding: 0 3px 0 18px;
+ background-color: #dddddd;
+ border-radius: 4px;
+ margin: 4px 0 0 4px;
+ vertical-align: top;
+ border: 1px solid #d2d2d2;
+}
+
+.combo-search .combo-search-applied-filter i {
+ position: absolute;
+ left: 5px;
+ font-size: 12px;
+ top: 5px;
+ color: #999;
+ cursor: pointer;
+}
+
+.combo-search .combo-search-applied-filter i:hover {
+ color: #333;
+}
+
+.combo-search .combo-search-applied-filter span,
+.combo-search .combo-search-applied-filter input {
+ color: #666;
+ font-size: 11px;
+}
+
+.combo-search .combo-search-applied-filter span {
+ font-weight: bold;
+}
+
+.combo-search .combo-search-applied-filter .combo-search-input-wrapper {
+ height: 22px;
+ margin-left: 0;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css b/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css
index b4aa558..8f693a8 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css
@@ -1331,3 +1331,39 @@ th.entity-actions {
.entity-actions a:focus:hover {
text-decoration: none;
}
+
+.search-box-button {
+ position: relative;
+ margin-right: 5px;
+}
+
+.search-box-button .btn {
+ padding: 10px;
+}
+
+.search-box-row {
+ padding-top: 15px;
+ padding-bottom: 5px;
+}
+
+.popup-arrow-up {
+ background: inherit;
+ z-index: 1;
+ left: 6px;
+ position: absolute;
+ width: 24px;
+ height: 16px;
+ overflow: hidden;
+}
+
+.popup-arrow-up:after {
+ content: "";
+ position: absolute;
+ width: 20px;
+ height: 20px;
+ background: #fff;
+ transform: rotate(45deg);
+ top: 10px;
+ left: 2px;
+ box-shadow: -2px -2px 10px -3px rgba(0, 0, 0, 0.5);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
index ae57b86..9e9cb55 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
@@ -24,11 +24,21 @@
{{'views.create' | translate}}
</button>
</div>
+ <div class="search-box-button pull-right">
+ <button class="btn btn-default" ng-click="toggleSearchBox()">
+ <i class="fa fa-filter" aria-hidden="true"></i>
+ </button>
+ <div class="popup-arrow-up hide"></div>
+ </div>
+ </div>
+
+ <div class="search-box-row hide">
+ <combo-search suggestions="filters" filter-change="filterInstances" placeholder="Search"></combo-search>
</div>
<table class="table table-striped table-hover">
<thead>
- <tr class="fix-bottom">
+ <tr>
<th class="col-md-2">
<span>{{'common.name' | translate}}</span>
</th>
@@ -45,39 +55,6 @@
<span>{{'common.actions' | translate}}</span>
</th>
</tr>
- <tr class="fix-top">
- <th>
- <div class="search-container">
- <input type="text" class="form-control" placeholder="{{'common.any' | translate}}"
- ng-model="instanceNameFilter" ng-change="filterInstances()">
- <button type="button" class="close clearfilter" ng-show="instanceNameFilter"
- ng-click="instanceNameFilter=''; filterInstances()">
- <span aria-hidden="true">×</span>
- <span class="sr-only">{{'common.controls.close' | translate}}</span>
- </button>
- </div>
- </th>
- <th>
- <div class="search-container">
- <input type="text" class="form-control" placeholder="{{'common.any' | translate}}"
- ng-model="instanceUrlFilter" ng-change="filterInstances()">
- <button type="button" class="close clearfilter" ng-show="instanceUrlFilter"
- ng-click="instanceUrlFilter=''; filterInstances()">
- <span aria-hidden="true">×</span>
- <span class="sr-only">{{'common.controls.close' | translate}}</span>
- </button>
- </div>
- </th>
- <th>
- <select class="form-control typefilter v-small-input"
- ng-model="instanceTypeFilter"
- ng-options="item.label for item in typeFilterOptions"
- ng-change="filterInstances()">
- </select>
- </th>
- <th></th>
- <th></th>
- </tr>
</thead>
<tbody>
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/views/directives/comboSearch.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/directives/comboSearch.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/directives/comboSearch.html
new file mode 100644
index 0000000..a4fdfc2
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/directives/comboSearch.html
@@ -0,0 +1,63 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<div class="combo-search">
+ <div class="combo-search-inner" ng-click="observeSearchFilterInput($event)">
+ <div class="combo-search-applied-filter" ng-repeat="filter in appliedFilters" ng-click="forceFocus($event, filter)">
+ <i class="fa fa-times-circle" ng-click="removeFilter(filter)"></i>
+ <span>{{filter.label}}:</span>
+ <div class="combo-search-input-wrapper">
+ <input type="text"
+ autocomplete="off"
+ ng-attr-name="{{filter.id}}"
+ class="combo-search-input"
+ ng-model="filter.searchOptionInput"
+ ng-change="observeSearchOptionInput(filter)"
+ ng-blur="hideAutocomplete(filter)"/>
+ <div class="combo-search-dropdown" ng-show="filter.showAutoComplete">
+ <ul>
+ <li ng-repeat="item in filter.filteredOptions" class="filter">
+ <a ng-click="selectOption($event, item, filter)"
+ ng-class="{active: item.active}"
+ ng-mouseover="makeActive(item, filter.filteredOptions)">{{item.label}}</a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ </div>
+ <div class="combo-search-input-wrapper">
+ <input type="text"
+ autocomplete="off"
+ placeholder="{{appliedFilters.length === 0 ? placeholder : ''}}"
+ class="combo-search-input main-input"
+ ng-model="searchFilterInput"
+ ng-change="observeSearchFilterInput()"/>
+ <div class="combo-search-dropdown" ng-show="showAutoComplete">
+ <ul>
+ <li ng-repeat="item in filterSuggestions"
+ ng-class="{category: item.isCategory, filter: !item.isCategory, hide: (item.isCategory && item.isDefault)}">
+ <a ng-click="selectFilter(item, $event)"
+ ng-class="{active: item.active}"
+ ng-mouseover="makeActive(item, filterSuggestions)">{{item.label}}</a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ <i class="combo-search-close fa fa-times-circle" ng-show="appliedFilters.length" ng-click="clearFilters()"></i>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
index 67d650e..7a8e6f4 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
@@ -61,7 +61,7 @@
<div class="alert empty-table-alert col-sm-12" ng-show="!remoteClusters.length && !isLoading">
{{'common.alerts.noRemoteClusterDisplay' | translate}}
</div>
- <div class="col-sm-12 table-bar">
+ <div class="col-sm-12 table-bar" ng-show="tableInfo.total >= minInstanceForPagination">
<div class="pull-left filtered-info">
<span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.groups} }}</span>
<span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
index 279343b..9d81543 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
@@ -123,7 +123,7 @@
<div class="alert empty-table-alert col-sm-12" ng-show="!repos.length && !isLoading">
{{'common.alerts.nothingToDisplay' | translate:{term: getConstant("common.version")} }}
</div>
- <div class="col-sm-12 table-bar">
+ <div class="col-sm-12 table-bar" ng-show="tableInfo.total >= minInstanceForPagination">
<div class="pull-left filtered-info">
<span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: getConstant("common.versions")} }}</span>
<span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
new file mode 100644
index 0000000..362b94a
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('#Cluster', function () {
+ describe('ViewsListCtrl', function() {
+ var scope, ctrl;
+
+ beforeEach(function () {
+ module('ambariAdminConsole');
+ inject(function($rootScope, $controller) {
+ scope = $rootScope.$new();
+ ctrl = $controller('ViewsListCtrl', {$scope: scope});
+ });
+ scope.instances = [
+ {
+ short_url_name: 'sun1',
+ url: 'url1',
+ view_name: 'vn1',
+ instance_name: 'in1',
+ short_url: 'su1'
+ },
+ {
+ short_url_name: 'sun2',
+ url: 'url2',
+ view_name: 'vn2',
+ instance_name: 'in2',
+ short_url: 'su2'
+ }
+ ];
+ });
+
+ describe('#initFilterOptions()', function () {
+ beforeEach(function() {
+ scope.initFilterOptions();
+ });
+
+ it('should fill short_url_name options', function() {
+ expect(scope.filters[0].options).toEqual([
+ {
+ key: 'sun1',
+ label: 'sun1'
+ },
+ {
+ key: 'sun2',
+ label: 'sun2'
+ }
+ ]);
+ });
+
+ it('should fill url options', function() {
+ expect(scope.filters[1].options).toEqual([
+ {
+ key: '/main/view/vn1/su1',
+ label: '/main/view/vn1/su1'
+ },
+ {
+ key: '/main/view/vn2/su2',
+ label: '/main/view/vn2/su2'
+ }
+ ]);
+ });
+
+ it('should fill view_name options', function() {
+ expect(scope.filters[2].options).toEqual([
+ {
+ key: 'vn1',
+ label: 'vn1'
+ },
+ {
+ key: 'vn2',
+ label: 'vn2'
+ }
+ ]);
+ });
+
+ it('should fill instance_name options', function() {
+ expect(scope.filters[3].options).toEqual([
+ {
+ key: 'in1',
+ label: 'in1'
+ },
+ {
+ key: 'in2',
+ label: 'in2'
+ }
+ ]);
+ });
+ });
+
+
+ describe('#filterInstances', function() {
+ beforeEach(function() {
+ spyOn(scope, 'resetPagination');
+ });
+
+ it('all should be filtered when filters not applied', function() {
+ scope.filterInstances();
+ expect(scope.tableInfo.filtered).toEqual(2);
+ scope.filterInstances([]);
+ expect(scope.tableInfo.filtered).toEqual(2);
+ });
+
+ it('resetPagination should be called', function() {
+ scope.filterInstances();
+ expect(scope.resetPagination).toHaveBeenCalled();
+ });
+
+ it('one view should be filtered', function() {
+ var appliedFilters = [
+ {
+ key: 'view_name',
+ values: ['vn1']
+ }
+ ];
+ scope.filterInstances(appliedFilters);
+ expect(scope.tableInfo.filtered).toEqual(1);
+ expect(scope.instances[0].isFiltered).toBeTruthy();
+ expect(scope.instances[1].isFiltered).toBeFalsy();
+ });
+
+ it('two views should be filtered', function() {
+ var appliedFilters = [
+ {
+ key: 'view_name',
+ values: ['vn1', 'vn2']
+ }
+ ];
+ scope.filterInstances(appliedFilters);
+ expect(scope.tableInfo.filtered).toEqual(2);
+ expect(scope.instances[0].isFiltered).toBeTruthy();
+ expect(scope.instances[1].isFiltered).toBeTruthy();
+ });
+
+ it('one views should be filtered with combo filter', function() {
+ var appliedFilters = [
+ {
+ key: 'view_name',
+ values: ['vn1', 'vn2']
+ },
+ {
+ key: 'instance_name',
+ values: ['in2']
+ }
+ ];
+ scope.filterInstances(appliedFilters);
+ expect(scope.tableInfo.filtered).toEqual(1);
+ expect(scope.instances[0].isFiltered).toBeFalsy();
+ expect(scope.instances[1].isFiltered).toBeTruthy();
+ });
+ });
+ });
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/30d89a33/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
new file mode 100644
index 0000000..9bc7083
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('#comboSearch', function () {
+ var scope, element;
+
+ beforeEach(module('ambariAdminConsole'));
+ beforeEach(module('views/directives/comboSearch.html'));
+
+ beforeEach(inject(function($rootScope, $compile) {
+ scope = $rootScope.$new();
+
+ var preCompiledElement = '<combo-search suggestions="filters" filter-change="filterItems" placeholder="Search"></combo-search>';
+
+ scope.filters = [
+ {
+ key: 'f1',
+ label: 'filter1',
+ options: []
+ },
+ {
+ key: 'f2',
+ label: 'filter2',
+ options: []
+ }
+ ];
+ scope.filterItems = angular.noop;
+ spyOn(scope, 'filterItems');
+
+
+ element = $compile(preCompiledElement)(scope);
+ scope.$digest();
+ }));
+
+ afterEach(function() {
+ element.remove();
+ });
+
+
+ describe('#removeFilter', function() {
+ it('should remove filter by id', function () {
+ var isoScope = element.isolateScope();
+ isoScope.appliedFilters.push({
+ id: 1
+ });
+ spyOn(isoScope, 'observeSearchFilterInput');
+ spyOn(isoScope, 'updateFilters');
+
+ isoScope.removeFilter({id: 1});
+
+ expect(isoScope.appliedFilters).toEqual([]);
+ expect(isoScope.observeSearchFilterInput).toHaveBeenCalled();
+ expect(isoScope.updateFilters).toHaveBeenCalledWith([]);
+ });
+ });
+
+ describe('#clearFilters', function() {
+ it('should empty appliedFilters', function () {
+ var isoScope = element.isolateScope();
+ isoScope.appliedFilters.push({
+ id: 1
+ });
+ spyOn(isoScope, 'updateFilters');
+
+ isoScope.clearFilters();
+
+ expect(isoScope.appliedFilters).toEqual([]);
+ expect(isoScope.updateFilters).toHaveBeenCalledWith([]);
+ });
+ });
+
+ describe('#selectFilter', function() {
+ it('should add new filter to appliedFilters', function () {
+ var isoScope = element.isolateScope();
+
+ isoScope.selectFilter({
+ key: 'f1',
+ label: 'filter1',
+ options: []
+ });
+
+ expect(isoScope.appliedFilters[0]).toEqual({
+ id: 'filter_1',
+ currentOption: null,
+ filteredOptions: [],
+ searchOptionInput: '',
+ key: 'f1',
+ label: 'filter1',
+ options: [],
+ showAutoComplete: false
+ });
+ expect(isoScope.isEditing).toBeFalsy();
+ expect(isoScope.showAutoComplete).toBeFalsy();
+ expect(isoScope.searchFilterInput).toEqual('');
+ });
+ });
+
+ describe('#selectOption', function() {
+ it('should set value to appliedFilter', function () {
+ var isoScope = element.isolateScope();
+ var filter = {};
+
+ spyOn(isoScope, 'observeSearchFilterInput');
+ spyOn(isoScope, 'updateFilters');
+
+ isoScope.selectOption(null, {
+ key: 'o1',
+ label: 'option1'
+ }, filter);
+
+ expect(filter.currentOption).toEqual({
+ key: 'o1',
+ label: 'option1'
+ });
+ expect(filter.showAutoComplete).toBeFalsy();
+ expect(isoScope.observeSearchFilterInput).toHaveBeenCalled();
+ expect(isoScope.updateFilters).toHaveBeenCalled();
+ });
+ });
+
+ describe('#hideAutocomplete', function() {
+
+ it('showAutoComplete should be false when filter passed', function () {
+ var isoScope = element.isolateScope();
+ var filter = {
+ showAutoComplete: true
+ };
+ jasmine.Clock.useMock();
+
+ isoScope.hideAutocomplete(filter);
+
+ jasmine.Clock.tick(101);
+ expect(filter.showAutoComplete).toBeFalsy();
+ });
+
+ it('showAutoComplete should be false when isEditing = false', function () {
+ var isoScope = element.isolateScope();
+ jasmine.Clock.useMock();
+
+ isoScope.isEditing = false;
+ isoScope.showAutoComplete = true;
+ isoScope.hideAutocomplete();
+
+ jasmine.Clock.tick(101);
+ expect(isoScope.showAutoComplete).toBeFalsy();
+ });
+
+ it('showAutoComplete should be false when isEditing = true', function () {
+ var isoScope = element.isolateScope();
+ jasmine.Clock.useMock();
+
+ isoScope.isEditing = true;
+ isoScope.showAutoComplete = true;
+ isoScope.hideAutocomplete();
+
+ jasmine.Clock.tick(101);
+ expect(isoScope.showAutoComplete).toBeTruthy();
+ });
+ });
+
+ describe('#makeActive', function() {
+ it('category option can not be active', function () {
+ var isoScope = element.isolateScope();
+ var active = {
+ key: 'o1',
+ isCategory: true,
+ active: false
+ };
+
+ isoScope.makeActive(active, [active]);
+
+ expect(active.active).toBeFalsy();
+ });
+
+ it('value option can be active', function () {
+ var isoScope = element.isolateScope();
+ var active = {
+ key: 'o1',
+ isCategory: false,
+ active: false
+ };
+
+ isoScope.makeActive(active, [active]);
+
+ expect(active.active).toBeTruthy();
+ });
+ });
+
+ describe('#updateFilters', function() {
+ it('filter function from parent scope should be called', function () {
+ var isoScope = element.isolateScope();
+ spyOn(isoScope, 'extractFilters').andReturn([{}]);
+
+ isoScope.updateFilters([{}]);
+
+ expect(scope.filterItems).toHaveBeenCalledWith([{}]);
+ });
+ });
+
+ describe('#extractFilters', function() {
+ it('should extract filters', function () {
+ var isoScope = element.isolateScope();
+ var filters = [
+ {
+ currentOption: { key: 'o1'},
+ key: 'f1'
+ },
+ {
+ currentOption: { key: 'o2'},
+ key: 'f1'
+ },
+ {
+ currentOption: null,
+ key: 'f2'
+ }
+ ];
+
+ expect(isoScope.extractFilters(filters)).toEqual([
+ {
+ key: 'f1',
+ values: ['o1', 'o2']
+ }
+ ]);
+ });
+ });
+
+});
[50/51] [abbrv] ambari git commit: AMBARI-22449. Improved
service/component dependency support (amagyar)
Posted by am...@apache.org.
AMBARI-22449. Improved service/component dependency support (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/683574c3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/683574c3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/683574c3
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 683574c348801f99142f3f01470c75f332d6001f
Parents: 6a98d34
Author: Attila Magyar <am...@hortonworks.com>
Authored: Thu Nov 16 12:27:13 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../app/controllers/wizard/step4_controller.js | 115 ++++++++++---------
ambari-web/app/messages.js | 3 +-
ambari-web/app/models/stack_service.js | 82 +++++++++++++
.../test/controllers/wizard/step4_test.js | 16 +--
4 files changed, 155 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/683574c3/ambari-web/app/controllers/wizard/step4_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step4_controller.js b/ambari-web/app/controllers/wizard/step4_controller.js
index d82bf34..d39164b 100644
--- a/ambari-web/app/controllers/wizard/step4_controller.js
+++ b/ambari-web/app/controllers/wizard/step4_controller.js
@@ -386,61 +386,29 @@ App.WizardStep4Controller = Em.ArrayController.extend({
*/
serviceDependencyValidation: function(callback) {
var selectedServices = this.filterProperty('isSelected', true);
+ var availableServices = this.get('content');
var missingDependencies = [];
- var missingDependenciesDisplayName = [];
selectedServices.forEach(function(service) {
- var requiredServices = service.get('requiredServices');
- if (!!requiredServices && requiredServices.length) {
- requiredServices.forEach(function(_requiredService){
- var requiredService = this.findProperty('serviceName', _requiredService);
- if (requiredService) {
- if(requiredService.get('isSelected') === false) {
- if(missingDependencies.indexOf(_requiredService) === -1) {
- missingDependencies.push(_requiredService);
- missingDependenciesDisplayName.push(requiredService.get('displayNameOnSelectServicePage'));
- }
- }
- else {
- //required service is selected, remove the service error from errorObject array
- var serviceName = requiredService.get('serviceName');
- var serviceError = this.get('errorStack').filterProperty('id',"serviceCheck_"+serviceName);
- if(serviceError) {
- this.get('errorStack').removeObject(serviceError[0]);
- }
- }
- }
- },this);
- }
- },this);
+ service.collectMissingDependencies(selectedServices, availableServices, missingDependencies);
+ });
+ this.cleanExistingServiceCheckErrors();
+ this.addServiceCheckErrors(missingDependencies, callback);
+ },
- //create a copy of the errorStack, reset it
- //and add the dependencies in the correct order
- var errorStackCopy = this.get('errorStack');
- this.set('errorStack', []);
+ cleanExistingServiceCheckErrors() {
+ var existingServiceCheckErrors = this.get('errorStack').filter(function (error) {
+ return error.id.startsWith('serviceCheck_');
+ });
+ this.get('errorStack').removeObjects(existingServiceCheckErrors);
+ },
- if (missingDependencies.length > 0) {
- for(var i = 0; i < missingDependencies.length; i++) {
- this.addValidationError({
- id: 'serviceCheck_' + missingDependencies[i],
- callback: this.needToAddServicePopup,
- callbackParams: [{serviceName: missingDependencies[i], selected: true}, 'serviceCheck', missingDependenciesDisplayName[i], callback]
- });
- }
- }
-
- //iterate through the errorStackCopy array and add to errorStack array, the error objects that have no matching entry in the errorStack
- //and that are not related to serviceChecks since serviceCheck errors have already been added when iterating through the missing dependencies list
- //Only add Ranger, Ambari Metrics, Spark and file system service validation errors if they exist in the errorStackCopy array
- var ctr = 0;
- while(ctr < errorStackCopy.length) {
- //no matching entry in errorStack array
- if (!this.get('errorStack').someProperty('id', errorStackCopy[ctr].id)) {
- //not serviceCheck error
- if(!errorStackCopy[ctr].id.startsWith('serviceCheck_')) {
- this.get('errorStack').push(this.createError(errorStackCopy[ctr]));
- }
- }
- ctr++;
+ addServiceCheckErrors(missingDependencies, callback) {
+ for(var i = 0; i < missingDependencies.length; i++) {
+ this.addValidationError({
+ id: 'serviceCheck_' + missingDependencies[i].get('serviceName'),
+ callback: this.needToAddMissingDependency,
+ callbackParams: [missingDependencies[i], 'serviceCheck', callback]
+ });
}
},
@@ -486,7 +454,7 @@ App.WizardStep4Controller = Em.ArrayController.extend({
return App.ModalPopup.show({
'data-qa': 'need-add-service-confirmation-modal',
header: Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.header').format(serviceName),
- body: Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.body').format(serviceName),
+ body: Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.body').format(serviceName, serviceName),
onPrimary: function () {
Em.makeArray(services).forEach(function (service) {
self.findProperty('serviceName', service.serviceName).set('isSelected', service.selected);
@@ -509,6 +477,49 @@ App.WizardStep4Controller = Em.ArrayController.extend({
});
},
+ needToAddMissingDependency: function (missingDependency, i18nSuffix, callback, id) {
+ var self = this;
+ var displayName = missingDependency.get('displayName');
+ if (missingDependency.hasMultipleOptions()) {
+ return this.showDependencyPopup(
+ id,
+ Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.header').format(displayName),
+ Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.body2').format(displayName, missingDependency.displayOptions()),
+ callback
+ );
+ } else {
+ return this.showDependencyPopup(
+ id,
+ Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.header').format(displayName),
+ Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.body').format(displayName, missingDependency.get('serviceName')),
+ callback,
+ function () {
+ missingDependency.selectFirstCompatible();
+ self.onPrimaryPopupCallback(callback, id);
+ this.hide();
+ }
+ );
+ }
+ },
+
+ showDependencyPopup: function(id, header, body, callback, primaryAction) {
+ return App.ModalPopup.show({
+ 'data-qa': 'need-add-service-confirmation-modal',
+ header: header,
+ body: body,
+ onPrimary: primaryAction || function() { this.onClose(); },
+ onSecondary: function() {
+ this.onClose();
+ },
+ onClose: function() {
+ if (callback) {
+ callback(id);
+ }
+ this._super();
+ }
+ });
+ },
+
/**
* Show popup with info about not selected service
* @param {function} callback
http://git-wip-us.apache.org/repos/asf/ambari/blob/683574c3/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 57999a8..f498a58 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -882,7 +882,8 @@ Em.I18n.translations = {
'installer.step4.multipleDFS.popup.header':'Multiple File Systems Selected',
'installer.step4.multipleDFS.popup.body':'You selected more than one file system. We will automatically select only {0}. Is this OK?',
'installer.step4.serviceCheck.popup.header':'{0} Needed',
- 'installer.step4.serviceCheck.popup.body':'You did not select {0}, but it is needed by other services you selected. We will automatically add {0}. Is this OK?',
+ 'installer.step4.serviceCheck.popup.body':'You did not select {0}, but it is needed by other services you selected. We will automatically add {1}. Is this OK?',
+ 'installer.step4.serviceCheck.popup.body2':'You did not select {0}, but it is needed by other services you selected. Select a compatible service from the following list: {1}',
'installer.step4.limitedFunctionality.popup.header':'Limited Functionality Warning',
'installer.step4.ambariMetricsCheck.popup.body':'Ambari Metrics collects metrics from the cluster and makes them available to Ambari. If you do not install Ambari Metrics service, metrics will not be accessible from Ambari. Are you sure you want to proceed without Ambari Metrics?',
'installer.step4.ambariInfraCheck.popup.body':'Since Ambari Infra is not selected, you must supply your own Solr to make Atlas work. Are you sure you want to proceed?',
http://git-wip-us.apache.org/repos/asf/ambari/blob/683574c3/ambari-web/app/models/stack_service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js
index 241f6ab..46dfee5 100644
--- a/ambari-web/app/models/stack_service.js
+++ b/ambari-web/app/models/stack_service.js
@@ -21,6 +21,56 @@ var stringUtils = require('utils/string_utils');
require('utils/helper');
require('models/configs/objects/service_config_category');
+var Dependency = Ember.Object.extend({
+ name: Ember.computed('service', function() {
+ return this.get('service').get('serviceName');
+ }),
+
+ displayName: Ember.computed('service', function() {
+ return this.get('service').get('displayNameOnSelectServicePage');
+ }),
+
+ compatibleServices: function(services) {
+ return services.filterProperty('serviceName', this.get('name'));
+ },
+
+ isMissing: function(selectedServices) {
+ return Em.isEmpty(this.compatibleServices(selectedServices));
+ }
+});
+
+var HcfsDependency = Dependency.extend({
+ displayName: Ember.computed('service', function() {
+ return 'a Hadoop Compatible File System';
+ }),
+
+ compatibleServices: function(services) {
+ return services.filterProperty("isDFS", true);
+ }
+});
+
+Dependency.reopenClass({
+ fromService: function(service) {
+ return service.get('isDFS')
+ ? HcfsDependency.create({service: service})
+ : Dependency.create({service: service});
+ }
+});
+
+var MissingDependency = Ember.Object.extend({
+ hasMultipleOptions: function() {
+ return this.get('compatibleServices').length > 1;
+ },
+
+ selectFirstCompatible: function() {
+ this.get('compatibleServices')[0].set('isSelected', true);
+ },
+
+ displayOptions: function() {
+ return this.get('compatibleServices').mapProperty('serviceName').join(', ');
+ }
+});
+
/**
* This model loads all services supported by the stack
* The model maps to the http://hostname:8080/api/v1/stacks/HDP/versions/${versionNumber}/services?fields=StackServices/*,serviceComponents/*
@@ -71,6 +121,38 @@ App.StackService = DS.Model.extend({
*/
dependentServiceNames: DS.attr('array', {defaultValue: []}),
+ dependencies: function(availableServices) {
+ return this.get('requiredServices')
+ .map(function (serviceName) { return availableServices.findProperty('serviceName', serviceName)})
+ .filter(function (each) { return !!each })
+ .map(function (service) { return Dependency.fromService(service); });
+ },
+
+ /**
+ * Add dependencies which are not already included in selectedServices to the given missingDependencies collection
+ */
+ collectMissingDependencies: function(selectedServices, availableServices, missingDependencies) {
+ this._missingDependencies(selectedServices, availableServices).forEach(function (dependency) {
+ this._addMissingDependency(dependency, availableServices, missingDependencies);
+ }.bind(this));
+ },
+
+ _missingDependencies: function(selectedServices, availableServices) {
+ return this.dependencies(availableServices).filter(function(dependency) {
+ return dependency.isMissing(selectedServices);
+ });
+ },
+
+ _addMissingDependency: function(dependency, availableServices, missingDependencies) {
+ if(!missingDependencies.some(function(each) { return each.get('serviceName') == dependency.get('name'); })) {
+ missingDependencies.push(MissingDependency.create({
+ serviceName: dependency.get('name'),
+ displayName: dependency.get('displayName'),
+ compatibleServices: dependency.compatibleServices(availableServices)
+ }));
+ }
+ },
+
// Is the service a distributed filesystem
isDFS: function () {
return this.get('serviceType') === 'HCFS' || ['HDFS', 'GLUSTERFS'].contains(this.get('serviceName'));
http://git-wip-us.apache.org/repos/asf/ambari/blob/683574c3/ambari-web/test/controllers/wizard/step4_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/wizard/step4_test.js b/ambari-web/test/controllers/wizard/step4_test.js
index 0fad0c1..1cd4863 100644
--- a/ambari-web/test/controllers/wizard/step4_test.js
+++ b/ambari-web/test/controllers/wizard/step4_test.js
@@ -32,7 +32,7 @@ describe('App.WizardStep4Controller', function () {
beforeEach(function() {
controller = App.WizardStep4Controller.create();
services.forEach(function(serviceName) {
- controller.pushObject(Ember.Object.create({
+ controller.pushObject(App.StackService.createRecord({
'serviceName':serviceName, 'isSelected': true, 'isHiddenOnSelectServicePage': false, 'isInstalled': false, 'isDisabled': 'HDFS' === serviceName, isDFS: 'HDFS' === serviceName
}));
});
@@ -43,7 +43,7 @@ describe('App.WizardStep4Controller', function () {
modelSetup.setupStackServiceComponent();
if (selectedServiceNames.contains('GLUSTERFS')) allServices.push('GLUSTERFS');
allServices = allServices.map(function(serviceName) {
- return [Ember.Object.create({
+ return [App.StackService.createRecord({
'serviceName': serviceName,
'isSelected': false,
'canBeSelected': true,
@@ -51,7 +51,7 @@ describe('App.WizardStep4Controller', function () {
isPrimaryDFS: serviceName === 'HDFS',
isDFS: ['HDFS','GLUSTERFS'].contains(serviceName),
isMonitoringService: ['GANGLIA'].contains(serviceName),
- requiredServices: App.StackService.find(serviceName).get('requiredServices'),
+ requiredServices: App.StackService.find(serviceName).get('requiredServices') || [],
displayNameOnSelectServicePage: App.format.role(serviceName, true),
coSelectedServices: function() {
return App.StackService.coSelected[this.get('serviceName')] || [];
@@ -167,7 +167,7 @@ describe('App.WizardStep4Controller', function () {
beforeEach(function () {
controller.clear();
Object.keys(testCase.condition).forEach(function (id) {
- controller.pushObject(Ember.Object.create({
+ controller.pushObject(App.StackService.createRecord({
serviceName: id,
isSelected: testCase.condition[id],
canBeSelected: true,
@@ -1006,7 +1006,7 @@ describe('App.WizardStep4Controller', function () {
});
it('serviceValidation should not be called when service not selected', function() {
- controller.pushObject(Em.Object.create({
+ controller.pushObject(App.StackService.createRecord({
serviceName: 'S1',
isSelected: false
}));
@@ -1016,7 +1016,7 @@ describe('App.WizardStep4Controller', function () {
it('serviceValidation should not be called when dependent service does not exist', function() {
controller.pushObjects([
- Em.Object.create({
+ App.StackService.createRecord({
serviceName: 'S1',
isSelected: true
})
@@ -1027,7 +1027,7 @@ describe('App.WizardStep4Controller', function () {
it('serviceValidation should not be called when dependent service is selected', function() {
controller.pushObjects([
- Em.Object.create({
+ App.StackService.createRecord({
serviceName: 'S1',
isSelected: true
}),
@@ -1042,7 +1042,7 @@ describe('App.WizardStep4Controller', function () {
it('serviceValidation should be called when dependent service is not selected', function() {
controller.pushObjects([
- Em.Object.create({
+ App.StackService.createRecord({
serviceName: 'S1',
isSelected: true
}),
[31/51] [abbrv] ambari git commit: AMBARI-22530. Refactor internal
code of handling info between kerberos wizard actions (echekanskiy)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/67fc4a37/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index ab85aa1..c7b69f0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -60,11 +60,14 @@ import org.apache.ambari.server.controller.internal.RequestStageContainer;
import org.apache.ambari.server.controller.utilities.KerberosChecker;
import org.apache.ambari.server.metadata.RoleCommandOrder;
import org.apache.ambari.server.orm.dao.ArtifactDAO;
+import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
import org.apache.ambari.server.orm.entities.ArtifactEntity;
+import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.security.credential.Credential;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.security.encryption.CredentialStoreService;
@@ -79,7 +82,6 @@ import org.apache.ambari.server.serveraction.kerberos.FinalizeKerberosServerActi
import org.apache.ambari.server.serveraction.kerberos.KDCType;
import org.apache.ambari.server.serveraction.kerberos.KerberosAdminAuthenticationException;
import org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileWriter;
-import org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileWriterFactory;
import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
import org.apache.ambari.server.serveraction.kerberos.KerberosKDCConnectionException;
import org.apache.ambari.server.serveraction.kerberos.KerberosKDCSSLConnectionException;
@@ -95,6 +97,7 @@ import org.apache.ambari.server.serveraction.kerberos.PrepareEnableKerberosServe
import org.apache.ambari.server.serveraction.kerberos.PrepareKerberosIdentitiesServerAction;
import org.apache.ambari.server.serveraction.kerberos.UpdateKerberosConfigsServerAction;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.stageplanner.RoleGraph;
import org.apache.ambari.server.stageplanner.RoleGraphFactory;
import org.apache.ambari.server.state.Cluster;
@@ -129,14 +132,17 @@ import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.directory.server.kerberos.shared.keytab.Keytab;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Singleton;
@@ -200,19 +206,19 @@ public class KerberosHelperImpl implements KerberosHelper {
private KerberosDescriptorFactory kerberosDescriptorFactory;
@Inject
- private KerberosIdentityDataFileWriterFactory kerberosIdentityDataFileWriterFactory;
+ private ArtifactDAO artifactDAO;
@Inject
private KerberosPrincipalDAO kerberosPrincipalDAO;
@Inject
- private ArtifactDAO artifactDAO;
+ private KerberosKeytabDAO kerberosKeytabDAO;
@Inject
- private KerberosKeytabDAO kerberosKeytabDAO;
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
@Inject
- KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
+ private HostDAO hostDAO;
/**
* The injector used to create new instances of helper classes like CreatePrincipalsServerAction
@@ -234,7 +240,7 @@ public class KerberosHelperImpl implements KerberosHelper {
public RequestStageContainer toggleKerberos(Cluster cluster, SecurityType securityType,
RequestStageContainer requestStageContainer,
Boolean manageIdentities)
- throws AmbariException, KerberosOperationException {
+ throws AmbariException, KerberosOperationException {
KerberosDetails kerberosDetails = getKerberosDetails(cluster, manageIdentities);
@@ -258,7 +264,7 @@ public class KerberosHelperImpl implements KerberosHelper {
public RequestStageContainer executeCustomOperations(Cluster cluster, Map<String, String> requestProperties,
RequestStageContainer requestStageContainer,
Boolean manageIdentities)
- throws AmbariException, KerberosOperationException {
+ throws AmbariException, KerberosOperationException {
if (requestProperties != null) {
@@ -279,7 +285,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Set<String>> serviceComponentFilter = parseComponentFilter(requestProperties);
boolean updateConfigurations = !requestProperties.containsKey(DIRECTIVE_IGNORE_CONFIGS)
- || !"true".equalsIgnoreCase(requestProperties.get(DIRECTIVE_IGNORE_CONFIGS));
+ || !"true".equalsIgnoreCase(requestProperties.get(DIRECTIVE_IGNORE_CONFIGS));
boolean forceAllHosts = (hostFilter == null) || (hostFilter.contains("*"));
@@ -291,7 +297,7 @@ public class KerberosHelperImpl implements KerberosHelper {
if (handler != null) {
requestStageContainer = handle(cluster, getKerberosDetails(cluster, manageIdentities),
- serviceComponentFilter, hostFilter, null, null, requestStageContainer, handler);
+ serviceComponentFilter, hostFilter, null, null, requestStageContainer, handler);
} else {
throw new AmbariException(String.format("Unexpected directive value: %s", value));
}
@@ -353,19 +359,19 @@ public class KerberosHelperImpl implements KerberosHelper {
public RequestStageContainer ensureIdentities(Cluster cluster, Map<String, ? extends Collection<String>> serviceComponentFilter,
Set<String> hostFilter, Collection<String> identityFilter, Set<String> hostsToForceKerberosOperations,
RequestStageContainer requestStageContainer, Boolean manageIdentities)
- throws AmbariException, KerberosOperationException {
+ throws AmbariException, KerberosOperationException {
return handle(cluster, getKerberosDetails(cluster, manageIdentities), serviceComponentFilter, hostFilter, identityFilter,
- hostsToForceKerberosOperations, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.DEFAULT, false, false,
- false));
+ hostsToForceKerberosOperations, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.DEFAULT, false, false,
+ false));
}
@Override
public RequestStageContainer deleteIdentities(Cluster cluster, Map<String, ? extends Collection<String>> serviceComponentFilter,
Set<String> hostFilter, Collection<String> identityFilter, RequestStageContainer requestStageContainer,
Boolean manageIdentities)
- throws AmbariException, KerberosOperationException {
+ throws AmbariException, KerberosOperationException {
return handle(cluster, getKerberosDetails(cluster, manageIdentities), serviceComponentFilter, hostFilter, identityFilter, null,
- requestStageContainer, new DeletePrincipalsAndKeytabsHandler());
+ requestStageContainer, new DeletePrincipalsAndKeytabsHandler());
}
/**
@@ -383,23 +389,23 @@ public class KerberosHelperImpl implements KerberosHelper {
RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
DeleteIdentityHandler handler = new DeleteIdentityHandler(customCommandExecutionHelper, configuration.getDefaultServerTaskTimeout(), stageFactory, ambariManagementController);
DeleteIdentityHandler.CommandParams commandParameters = new DeleteIdentityHandler.CommandParams(
- components,
- identities,
- ambariManagementController.getAuthName(),
- dataDirectory,
- kerberosDetails.getDefaultRealm(),
- kerberosDetails.getKdcType());
+ components,
+ identities,
+ ambariManagementController.getAuthName(),
+ dataDirectory,
+ kerberosDetails.getDefaultRealm(),
+ kerberosDetails.getKdcType());
OrderedRequestStageContainer stageContainer = new OrderedRequestStageContainer(
- roleGraphFactory,
- roleCommandOrder,
- new RequestStageContainer(actionManager.getNextRequestId(), null, requestFactory, actionManager));
+ roleGraphFactory,
+ roleCommandOrder,
+ new RequestStageContainer(actionManager.getNextRequestId(), null, requestFactory, actionManager));
handler.addDeleteIdentityStages(cluster, stageContainer, commandParameters, kerberosDetails.manageIdentities());
stageContainer.getRequestStageContainer().persist();
}
@Override
public void configureServices(Cluster cluster, Map<String, Collection<String>> serviceFilter)
- throws AmbariException, KerberosInvalidConfigurationException {
+ throws AmbariException, KerberosInvalidConfigurationException {
final Map<String, Set<String>> installedServices = new HashMap<>();
final Set<String> previouslyExistingServices = new HashSet<>();
@@ -407,40 +413,40 @@ public class KerberosHelperImpl implements KerberosHelper {
// We can create the map in the "shouldIncludeCommand" Command to avoid having to iterate
// over the returned ServiceComponentHost List.
getServiceComponentHosts(cluster,
- new Command<Boolean, ServiceComponentHost>() {
- @Override
- public Boolean invoke(ServiceComponentHost sch) throws AmbariException {
- if (sch != null) {
- String serviceName = sch.getServiceName();
-
- Set<String> installedComponents = installedServices.get(serviceName);
- if (installedComponents == null) {
- installedComponents = new HashSet<>();
- installedServices.put(serviceName, installedComponents);
- }
- installedComponents.add(sch.getServiceComponentName());
+ new Command<Boolean, ServiceComponentHost>() {
+ @Override
+ public Boolean invoke(ServiceComponentHost sch) throws AmbariException {
+ if (sch != null) {
+ String serviceName = sch.getServiceName();
- // Determine if this component was PREVIOUSLY installed, which implies that its containing service was PREVIOUSLY installed
- if (!previouslyExistingServices.contains(serviceName) && PREVIOUSLY_INSTALLED_STATES.contains(sch.getState())) {
- previouslyExistingServices.add(serviceName);
- }
+ Set<String> installedComponents = installedServices.get(serviceName);
+ if (installedComponents == null) {
+ installedComponents = new HashSet<>();
+ installedServices.put(serviceName, installedComponents);
+ }
+ installedComponents.add(sch.getServiceComponentName());
- return true;
+ // Determine if this component was PREVIOUSLY installed, which implies that its containing service was PREVIOUSLY installed
+ if (!previouslyExistingServices.contains(serviceName) && PREVIOUSLY_INSTALLED_STATES.contains(sch.getState())) {
+ previouslyExistingServices.add(serviceName);
}
- return false;
+ return true;
}
- });
+
+ return false;
+ }
+ });
Map<String, Map<String, String>> existingConfigurations = calculateExistingConfigurations(cluster, null);
Map<String, Map<String, String>> updates = getServiceConfigurationUpdates(cluster,
- existingConfigurations, installedServices, serviceFilter, previouslyExistingServices, true, true);
+ existingConfigurations, installedServices, serviceFilter, previouslyExistingServices, true, true);
// Store the updates...
for (Map.Entry<String, Map<String, String>> entry : updates.entrySet()) {
configHelper.updateConfigType(cluster, cluster.getDesiredStackVersion(),
- ambariManagementController, entry.getKey(), entry.getValue(), null,
- ambariManagementController.getAuthName(), "Enabling Kerberos for added components");
+ ambariManagementController, entry.getKey(), entry.getValue(), null,
+ ambariManagementController.getAuthName(), "Enabling Kerberos for added components");
}
}
@@ -452,7 +458,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Set<String> previouslyExistingServices,
boolean kerberosEnabled,
boolean applyStackAdvisorUpdates)
- throws KerberosInvalidConfigurationException, AmbariException {
+ throws KerberosInvalidConfigurationException, AmbariException {
Map<String, Map<String, String>> kerberosConfigurations = new HashMap<>();
KerberosDetails kerberosDetails = getKerberosDetails(cluster, null);
@@ -460,14 +466,14 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
- deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+ deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
Map<String, Set<String>> propertiesToIgnore = new HashMap<>();
// If Ambari is managing it own identities then add AMBARI to the set of installed service so
// that its Kerberos descriptor entries will be included.
if (createAmbariIdentities(existingConfigurations.get(KERBEROS_ENV))) {
- installedServices = new HashMap<>(installedServices);
+ installedServices = new HashMap<String, Set<String>>(installedServices);
installedServices.put(RootService.AMBARI.name(), Collections.singleton(RootComponent.AMBARI_SERVER.name()));
}
@@ -504,7 +510,7 @@ public class KerberosHelperImpl implements KerberosHelper {
processIdentityConfigurations(identityConfigurations, kerberosConfigurations, configurations, propertiesToIgnore);
mergeConfigurations(kerberosConfigurations,
- componentDescriptor.getConfigurations(!servicePreviouslyExisted), configurations, null);
+ componentDescriptor.getConfigurations(!servicePreviouslyExisted), configurations, null);
}
}
}
@@ -516,9 +522,9 @@ public class KerberosHelperImpl implements KerberosHelper {
setAuthToLocalRules(cluster, kerberosDescriptor, kerberosDetails.getDefaultRealm(), installedServices, configurations, kerberosConfigurations, false);
return (applyStackAdvisorUpdates)
- ? applyStackAdvisorUpdates(cluster, installedServices.keySet(), configurations, kerberosConfigurations, propertiesToIgnore,
- new HashMap<>(), kerberosEnabled)
- : kerberosConfigurations;
+ ? applyStackAdvisorUpdates(cluster, installedServices.keySet(), configurations, kerberosConfigurations, propertiesToIgnore,
+ new HashMap<>(), kerberosEnabled)
+ : kerberosConfigurations;
}
/**
@@ -536,7 +542,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Set<String> services,
Set<String> componentFilter,
Map<String, Map<String, String>> configurations)
- throws AmbariException {
+ throws AmbariException {
StackId stackVersion = cluster.getCurrentStackVersion();
List<String> hostNames = new ArrayList<>();
@@ -548,12 +554,12 @@ public class KerberosHelperImpl implements KerberosHelper {
}
StackAdvisorRequest request = StackAdvisorRequest.StackAdvisorRequestBuilder
- .forStack(stackVersion.getStackName(), stackVersion.getStackVersion())
- .forServices(services)
- .forHosts(hostNames)
- .withComponentHostsMap(cluster.getServiceComponentHostMap(null, services))
- .ofType(StackAdvisorRequest.StackAdvisorRequestType.HOST_GROUPS)
- .build();
+ .forStack(stackVersion.getStackName(), stackVersion.getStackVersion())
+ .forServices(services)
+ .forHosts(hostNames)
+ .withComponentHostsMap(cluster.getServiceComponentHostMap(null, services))
+ .ofType(StackAdvisorRequest.StackAdvisorRequestType.HOST_GROUPS)
+ .build();
try {
RecommendationResponse response = stackAdvisorHelper.recommend(request);
@@ -727,13 +733,13 @@ public class KerberosHelperImpl implements KerberosHelper {
}
StackAdvisorRequest request = StackAdvisorRequest.StackAdvisorRequestBuilder
- .forStack(stackId.getStackName(), stackId.getStackVersion())
- .forServices(services)
- .forHosts(hostNames)
- .withComponentHostsMap(cluster.getServiceComponentHostMap(null, services))
- .withConfigurations(requestConfigurations)
- .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
- .build();
+ .forStack(stackId.getStackName(), stackId.getStackVersion())
+ .forServices(services)
+ .forHosts(hostNames)
+ .withComponentHostsMap(cluster.getServiceComponentHostMap(null, services))
+ .withConfigurations(requestConfigurations)
+ .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
+ .build();
try {
RecommendationResponse response = stackAdvisorHelper.recommend(request);
@@ -752,11 +758,11 @@ public class KerberosHelperImpl implements KerberosHelper {
Set<String> ignoreProperties = (propertiesToIgnore == null) ? null : propertiesToIgnore.get(configType);
addRecommendedPropertiesForConfigType(kerberosConfigurations, configType, recommendedConfigProperties,
- existingConfigProperties, kerberosConfigProperties, ignoreProperties);
+ existingConfigProperties, kerberosConfigProperties, ignoreProperties);
if (recommendedConfigPropertyAttributes != null) {
removeRecommendedPropertiesForConfigType(configType, recommendedConfigPropertyAttributes,
- existingConfigProperties, kerberosConfigurations, ignoreProperties, propertiesToRemove);
+ existingConfigProperties, kerberosConfigurations, ignoreProperties, propertiesToRemove);
}
}
}
@@ -793,8 +799,8 @@ public class KerberosHelperImpl implements KerberosHelper {
// add the config and property if it also does not exist in the existing configurations
if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
- "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
- configType, propertyName, recommendedValue);
+ "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+ configType, propertyName, recommendedValue);
if (kerberosConfigProperties == null) {
kerberosConfigProperties = new HashMap<>();
@@ -808,8 +814,8 @@ public class KerberosHelperImpl implements KerberosHelper {
if ((value == null) ? (recommendedValue != null) : !value.equals(recommendedValue)) {
// If the recommended value is a change, automatically change it.
LOG.debug("Updating Kerberos configuration based on StackAdvisor recommendation:" +
- "\n\tConfigType: {}\n\tProperty: {}\n\tOld Value: {}\n\tNew Value: {}",
- configType, propertyName, (value == null) ? "" : value, (recommendedValue == null) ? "" : recommendedValue);
+ "\n\tConfigType: {}\n\tProperty: {}\n\tOld Value: {}\n\tNew Value: {}",
+ configType, propertyName, (value == null) ? "" : value, (recommendedValue == null) ? "" : recommendedValue);
kerberosConfigProperties.put(propertyName, recommendedValue);
}
@@ -837,12 +843,12 @@ public class KerberosHelperImpl implements KerberosHelper {
// add to propertiesToBeRemoved map
Map<String, String> kerberosConfigProperties = kerberosConfigurations.get(configType);
if (((ignoreProperties == null) || !ignoreProperties.contains(propertyName)) &&
- ((kerberosConfigProperties == null) || kerberosConfigProperties.get(propertyName) == null) &&
- (existingConfigProperties != null && existingConfigProperties.containsKey(propertyName))) {
+ ((kerberosConfigProperties == null) || kerberosConfigProperties.get(propertyName) == null) &&
+ (existingConfigProperties != null && existingConfigProperties.containsKey(propertyName))) {
LOG.debug("Property to remove from configuration based on StackAdvisor recommendation:" +
- "\n\tConfigType: {}\n\tProperty: {}",
- configType, propertyName);
+ "\n\tConfigType: {}\n\tProperty: {}",
+ configType, propertyName);
// kerberosEnabled add property to propertiesToRemove, otherwise to kerberosConfigurations map
if (propertiesToRemove != null) {
@@ -866,7 +872,7 @@ public class KerberosHelperImpl implements KerberosHelper {
@Override
public boolean ensureHeadlessIdentities(Cluster cluster, Map<String, Map<String, String>> existingConfigurations, Set<String> services)
- throws KerberosInvalidConfigurationException, AmbariException {
+ throws KerberosInvalidConfigurationException, AmbariException {
KerberosDetails kerberosDetails = getKerberosDetails(cluster, null);
@@ -876,7 +882,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
- deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+ deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
Map<String, String> kerberosConfiguration = kerberosDetails.getKerberosEnvProperties();
KerberosOperationHandler kerberosOperationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
@@ -886,7 +892,7 @@ public class KerberosHelperImpl implements KerberosHelper {
kerberosOperationHandler.open(administratorCredential, kerberosDetails.getDefaultRealm(), kerberosConfiguration);
} catch (KerberosOperationException e) {
String message = String.format("Failed to process the identities, could not properly open the KDC operation handler: %s",
- e.getMessage());
+ e.getMessage());
LOG.error(message);
throw new AmbariException(message, e);
}
@@ -1040,15 +1046,27 @@ public class KerberosHelperImpl implements KerberosHelper {
if ((operationHandler != null) && operationHandler.createKeytabFile(keytab, tmpKeytabFile)) {
String ownerName = variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations);
String ownerAccess = keytabDescriptor.getOwnerAccess();
- boolean ownerWritable = "w".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
- boolean ownerReadable = "r".equalsIgnoreCase(ownerAccess) || "rw".equalsIgnoreCase(ownerAccess);
String groupName = variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations);
String groupAccess = keytabDescriptor.getGroupAccess();
- boolean groupWritable = "w".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
- boolean groupReadable = "r".equalsIgnoreCase(groupAccess) || "rw".equalsIgnoreCase(groupAccess);
- configureAmbariIdentitiesServerAction.installAmbariServerIdentity(principal, tmpKeytabFile.getAbsolutePath(), destKeytabFilePath,
- ownerName, ownerReadable, ownerWritable, groupName, groupReadable, groupWritable, null);
+ // TODO check if this reliable
+ String componentName = principal.contains(KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME)
+ ? "AMBARI_SERVER_SELF"
+ : RootComponent.AMBARI_SERVER.name();
+
+ ResolvedKerberosPrincipal resolvedKerberosPrincipal = new ResolvedKerberosPrincipal(
+ null,
+ hostname,
+ principal,
+ false,
+ null,
+ RootService.AMBARI.name(),
+ componentName,
+ destKeytabFilePath
+ );
+
+ configureAmbariIdentitiesServerAction.installAmbariServerIdentity(resolvedKerberosPrincipal, tmpKeytabFile.getAbsolutePath(), destKeytabFilePath,
+ ownerName, ownerAccess, groupName, groupAccess, null);
LOG.debug("Successfully created keytab file for {} at {}", principal, destKeytabFile.getAbsolutePath());
} else {
LOG.error("Failed to create keytab file for {} at {}", principal, destKeytabFile.getAbsolutePath());
@@ -1058,7 +1076,7 @@ public class KerberosHelperImpl implements KerberosHelper {
}
} catch (KerberosOperationException e) {
throw new AmbariException(String.format("Failed to create keytab file for %s at %s: %s:",
- principal, destKeytabFile.getAbsolutePath(), e.getLocalizedMessage()), e);
+ principal, destKeytabFile.getAbsolutePath(), e.getLocalizedMessage()), e);
}
} else {
LOG.error("No keytab data is available to create the keytab file for {} at {}", principal, destKeytabFile.getAbsolutePath());
@@ -1074,24 +1092,24 @@ public class KerberosHelperImpl implements KerberosHelper {
@Override
public RequestStageContainer createTestIdentity(Cluster cluster, Map<String, String> commandParamsStage,
RequestStageContainer requestStageContainer)
- throws KerberosOperationException, AmbariException {
+ throws KerberosOperationException, AmbariException {
return handleTestIdentity(cluster, getKerberosDetails(cluster, null), commandParamsStage, requestStageContainer,
- new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.DEFAULT, false, false, false));
+ new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.DEFAULT, false, false, false));
}
@Override
public RequestStageContainer deleteTestIdentity(Cluster cluster, Map<String, String> commandParamsStage,
RequestStageContainer requestStageContainer)
- throws KerberosOperationException, AmbariException {
+ throws KerberosOperationException, AmbariException {
requestStageContainer = handleTestIdentity(cluster, getKerberosDetails(cluster, null), commandParamsStage, requestStageContainer, new DeletePrincipalsAndKeytabsHandler());
return requestStageContainer;
}
@Override
public void validateKDCCredentials(Cluster cluster) throws KerberosMissingAdminCredentialsException,
- KerberosAdminAuthenticationException,
- KerberosInvalidConfigurationException,
- AmbariException {
+ KerberosAdminAuthenticationException,
+ KerberosInvalidConfigurationException,
+ AmbariException {
validateKDCCredentials(null, cluster);
}
@@ -1102,7 +1120,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Map<String, String>> existingConfigurations,
Map<String, Map<String, String>> kerberosConfigurations,
boolean includePreconfigureData)
- throws AmbariException {
+ throws AmbariException {
boolean processAuthToLocalRules = true;
Map<String, String> kerberosEnvProperties = existingConfigurations.get(KERBEROS_ENV);
@@ -1132,8 +1150,8 @@ public class KerberosHelperImpl implements KerberosHelper {
// Add in the default configurations for the services that need to be preconfigured. These
// configurations may be needed while calculating the auth-to-local rules.
Map<String, Map<String, String>> replacements = (includePreconfigureData)
- ? addConfigurationsForPreProcessedServices(deepCopy(existingConfigurations), cluster, kerberosDescriptor, false)
- : existingConfigurations;
+ ? addConfigurationsForPreProcessedServices(deepCopy(existingConfigurations), cluster, kerberosDescriptor, false)
+ : existingConfigurations;
// Process top-level identities
addIdentities(authToLocalBuilder, kerberosDescriptor.getIdentities(true, filterContext), null, replacements);
@@ -1157,8 +1175,8 @@ public class KerberosHelperImpl implements KerberosHelper {
// service has been explicitly added to the cluster
if (preconfigure || explicitlyAdded) {
LOG.info("Adding identities for service {} to auth to local mapping [{}]",
- serviceName,
- (explicitlyAdded) ? "explicit" : "preconfigured");
+ serviceName,
+ (explicitlyAdded) ? "explicit" : "preconfigured");
// Process the service-level Kerberos descriptor
addIdentities(authToLocalBuilder, serviceDescriptor.getIdentities(true, filterContext), null, replacements);
@@ -1233,20 +1251,19 @@ public class KerberosHelperImpl implements KerberosHelper {
}
kerberosConfiguration.put(propertyName,
- builder.generate(AuthToLocalBuilder.ConcatenationType.translate(m.group(3))));
+ builder.generate(AuthToLocalBuilder.ConcatenationType.translate(m.group(3))));
}
}
}
}
}
-
@Override
public List<ServiceComponentHost> getServiceComponentHostsToProcess(final Cluster cluster,
final KerberosDescriptor kerberosDescriptor,
final Map<String, ? extends Collection<String>> serviceComponentFilter,
final Collection<String> hostFilter)
- throws AmbariException {
+ throws AmbariException {
return getServiceComponentHosts(cluster, new Command<Boolean, ServiceComponentHost>() {
@Override
public Boolean invoke(ServiceComponentHost sch) throws AmbariException {
@@ -1288,7 +1305,7 @@ public class KerberosHelperImpl implements KerberosHelper {
*/
private List<ServiceComponentHost> getServiceComponentHosts(Cluster cluster,
Command<Boolean, ServiceComponentHost> shouldIncludeCommand)
- throws AmbariException {
+ throws AmbariException {
List<ServiceComponentHost> serviceComponentHostsToProcess = new ArrayList<>();
// Get the hosts in the cluster
Collection<Host> hosts = cluster.getHosts();
@@ -1318,7 +1335,7 @@ public class KerberosHelperImpl implements KerberosHelper {
@Override
public Set<String> getHostsWithValidKerberosClient(Cluster cluster)
- throws AmbariException {
+ throws AmbariException {
Set<String> hostsWithValidKerberosClient = new HashSet<>();
List<ServiceComponentHost> schKerberosClients = cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name());
@@ -1342,7 +1359,7 @@ public class KerberosHelperImpl implements KerberosHelper {
public KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
boolean evaluateWhenClauses, Collection<String> additionalServices,
boolean includePreconfigureData)
- throws AmbariException {
+ throws AmbariException {
// !!! FIXME in a per-service view, what does this become?
Set<StackId> stackIds = new HashSet<>();
@@ -1409,12 +1426,12 @@ public class KerberosHelperImpl implements KerberosHelper {
public KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
StackId stackId, boolean includePreconfigureData) throws AmbariException {
KerberosDescriptor stackDescriptor = (kerberosDescriptorType == KerberosDescriptorType.STACK || kerberosDescriptorType == KerberosDescriptorType.COMPOSITE)
- ? getKerberosDescriptorFromStack(stackId, includePreconfigureData)
- : null;
+ ? getKerberosDescriptorFromStack(stackId, includePreconfigureData)
+ : null;
KerberosDescriptor userDescriptor = (kerberosDescriptorType == KerberosDescriptorType.USER || kerberosDescriptorType == KerberosDescriptorType.COMPOSITE)
- ? getKerberosDescriptorUpdates(cluster)
- : null;
+ ? getKerberosDescriptorUpdates(cluster)
+ : null;
return combineKerberosDescriptors(stackDescriptor, userDescriptor);
}
@@ -1424,7 +1441,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, KerberosConfigurationDescriptor> updates,
Map<String, Map<String, String>> replacements,
Set<String> configurationTypeFilter)
- throws AmbariException {
+ throws AmbariException {
if ((updates != null) && !updates.isEmpty()) {
if (configurations == null) {
@@ -1453,7 +1470,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Map<String, String>> replacements,
Cluster cluster,
KerberosDescriptor kerberosDescriptor)
- throws AmbariException {
+ throws AmbariException {
// Ensure the Kerberos descriptor exists....
if (kerberosDescriptor == null) {
@@ -1500,7 +1517,7 @@ public class KerberosHelperImpl implements KerberosHelper {
String componentName, Map<String, Map<String, String>> kerberosConfigurations,
Map<String, Map<String, String>> configurations,
Map<String, ResolvedKerberosKeytab> resolvedKeytabs, String realm)
- throws IOException {
+ throws IOException {
int identitiesAdded = 0;
if (identities != null) {
@@ -1541,14 +1558,24 @@ public class KerberosHelperImpl implements KerberosHelper {
String evaluatedPrincipal = principal.replace("_HOST", hostname).replace("_REALM", realm);
ResolvedKerberosKeytab resolvedKeytab = new ResolvedKerberosKeytab(
- keytabFilePath,
- keytabFileOwnerName,
- keytabFileOwnerAccess,
- keytabFileGroupName,
- keytabFileGroupAccess,
- Sets.newHashSet(Pair.of(hostId, Pair.of(evaluatedPrincipal, principalType))),
- serviceName.equalsIgnoreCase(RootService.AMBARI.name()),
- componentName.equalsIgnoreCase("AMBARI_SERVER_SELF")
+ keytabFilePath,
+ keytabFileOwnerName,
+ keytabFileOwnerAccess,
+ keytabFileGroupName,
+ keytabFileGroupAccess,
+ Sets.newHashSet(new ResolvedKerberosPrincipal(
+ hostId,
+ hostname,
+ evaluatedPrincipal,
+ "service".equalsIgnoreCase(principalType),
+ null,
+ serviceName,
+ componentName,
+ keytabFilePath
+ )
+ ),
+ serviceName.equalsIgnoreCase(RootService.AMBARI.name()),
+ componentName.equalsIgnoreCase("AMBARI_SERVER_SELF")
);
if (resolvedKeytabs.containsKey(keytabFilePath)) {
ResolvedKerberosKeytab sameKeytab = resolvedKeytabs.get(keytabFilePath);
@@ -1557,58 +1584,58 @@ public class KerberosHelperImpl implements KerberosHelper {
String warnTemplate = "Keytab '{}' on host '{}' has different {}, originally set to '{}' and '{}:{}' has '{}', using '{}'";
if (!resolvedKeytab.getOwnerName().equals(sameKeytab.getOwnerName())) {
LOG.warn(warnTemplate,
- keytabFilePath, hostname, "owners", sameKeytab.getOwnerName(),
- serviceName, componentName, resolvedKeytab.getOwnerName(),
- sameKeytab.getOwnerName());
+ keytabFilePath, hostname, "owners", sameKeytab.getOwnerName(),
+ serviceName, componentName, resolvedKeytab.getOwnerName(),
+ sameKeytab.getOwnerName());
differentOwners = true;
}
if (!resolvedKeytab.getOwnerAccess().equals(sameKeytab.getOwnerAccess())) {
LOG.warn(warnTemplate,
- keytabFilePath, hostname, "owner access", sameKeytab.getOwnerAccess(),
- serviceName, componentName, resolvedKeytab.getOwnerAccess(),
- sameKeytab.getOwnerAccess());
+ keytabFilePath, hostname, "owner access", sameKeytab.getOwnerAccess(),
+ serviceName, componentName, resolvedKeytab.getOwnerAccess(),
+ sameKeytab.getOwnerAccess());
}
// TODO probably fail on group difference. Some services can inject its principals to same keytab, but
// TODO with different owners, so make sure that keytabs are accessible through group acls
// TODO this includes same group name and group 'r' mode
if (!resolvedKeytab.getGroupName().equals(sameKeytab.getGroupName())) {
- if(differentOwners) {
+ if (differentOwners) {
LOG.error(warnTemplate,
- keytabFilePath, hostname, "groups", sameKeytab.getGroupName(),
- serviceName, componentName, resolvedKeytab.getGroupName(),
- sameKeytab.getGroupName());
+ keytabFilePath, hostname, "groups", sameKeytab.getGroupName(),
+ serviceName, componentName, resolvedKeytab.getGroupName(),
+ sameKeytab.getGroupName());
} else {
LOG.warn(warnTemplate,
- keytabFilePath, hostname, "groups", sameKeytab.getGroupName(),
- serviceName, componentName, resolvedKeytab.getGroupName(),
- sameKeytab.getGroupName());
+ keytabFilePath, hostname, "groups", sameKeytab.getGroupName(),
+ serviceName, componentName, resolvedKeytab.getGroupName(),
+ sameKeytab.getGroupName());
}
}
if (!resolvedKeytab.getGroupAccess().equals(sameKeytab.getGroupAccess())) {
- if(differentOwners) {
+ if (differentOwners) {
if (!sameKeytab.getGroupAccess().contains("r")) {
LOG.error("Keytab '{}' on host '{}' referenced by multiple identities which have different owners," +
"but 'r' attribute missing for group. Make sure all users (that need this keytab) are in '{}' +" +
"group and keytab can be read by this group",
- keytabFilePath,
- hostname,
- sameKeytab.getGroupName()
+ keytabFilePath,
+ hostname,
+ sameKeytab.getGroupName()
);
}
LOG.error(warnTemplate,
- keytabFilePath, hostname, "group access", sameKeytab.getGroupAccess(),
- serviceName, componentName, resolvedKeytab.getGroupAccess(),
- sameKeytab.getGroupAccess());
+ keytabFilePath, hostname, "group access", sameKeytab.getGroupAccess(),
+ serviceName, componentName, resolvedKeytab.getGroupAccess(),
+ sameKeytab.getGroupAccess());
} else {
LOG.warn(warnTemplate,
- keytabFilePath, hostname, "group access", sameKeytab.getGroupAccess(),
- serviceName, componentName, resolvedKeytab.getGroupAccess(),
- sameKeytab.getGroupAccess());
+ keytabFilePath, hostname, "group access", sameKeytab.getGroupAccess(),
+ serviceName, componentName, resolvedKeytab.getGroupAccess(),
+ sameKeytab.getGroupAccess());
}
}
// end validating
// merge principal to keytab
- sameKeytab.getMappedPrincipals().addAll(resolvedKeytab.getMappedPrincipals());
+ sameKeytab.mergePrincipals(resolvedKeytab);
// ensure that keytab file on ambari-server host creating jass file
if (sameKeytab.isMustWriteAmbariJaasFile() || resolvedKeytab.isMustWriteAmbariJaasFile()) {
sameKeytab.setMustWriteAmbariJaasFile(true);
@@ -1620,24 +1647,24 @@ public class KerberosHelperImpl implements KerberosHelper {
} else {
resolvedKeytabs.put(keytabFilePath, resolvedKeytab);
LOG.info("Keytab {} owner:'{}:{}', group:'{}:{}' is defined", keytabFilePath,
- keytabFileOwnerName, keytabFileOwnerAccess, keytabFileGroupName, keytabFileGroupAccess);
+ keytabFileOwnerName, keytabFileOwnerAccess, keytabFileGroupName, keytabFileGroupAccess);
}
// Append an entry to the action data file builder...
// TODO obsolete, move to ResolvedKerberosKeytab
- if(kerberosIdentityDataFileWriter != null) {
+ if (kerberosIdentityDataFileWriter != null) {
kerberosIdentityDataFileWriter.writeRecord(
- hostname,
- serviceName,
- componentName,
- evaluatedPrincipal,
- principalType,
- keytabFilePath,
- keytabFileOwnerName,
- keytabFileOwnerAccess,
- keytabFileGroupName,
- keytabFileGroupAccess,
- "true");
+ hostname,
+ serviceName,
+ componentName,
+ evaluatedPrincipal,
+ principalType,
+ keytabFilePath,
+ keytabFileOwnerName,
+ keytabFileOwnerAccess,
+ keytabFileGroupName,
+ keytabFileGroupAccess,
+ "true");
}
// Add the principal-related configuration to the map of configurations
@@ -1660,14 +1687,14 @@ public class KerberosHelperImpl implements KerberosHelper {
KerberosDescriptor kerberosDescriptor,
boolean includePreconfigureData,
boolean calculateClusterHostInfo)
- throws AmbariException {
+ throws AmbariException {
Map<String, Map<String, String>> calculatedConfigurations = addAdditionalConfigurations(
- cluster,
- calculateExistingConfigurations(cluster, hostname),
- hostname,
- (kerberosDescriptor == null) ? null : kerberosDescriptor.getProperties());
+ cluster,
+ calculateExistingConfigurations(cluster, hostname),
+ hostname,
+ (kerberosDescriptor == null) ? null : kerberosDescriptor.getProperties());
if (includePreconfigureData) {
calculatedConfigurations = addConfigurationsForPreProcessedServices(calculatedConfigurations, cluster, kerberosDescriptor, calculateClusterHostInfo);
@@ -1689,7 +1716,7 @@ public class KerberosHelperImpl implements KerberosHelper {
String serviceName,
String componentName,
boolean replaceHostNames)
- throws AmbariException {
+ throws AmbariException {
if ((clusterName == null) || clusterName.isEmpty()) {
throw new IllegalArgumentException("Invalid argument, cluster name is required");
@@ -1708,7 +1735,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Config kerberosEnvConfig = cluster.getDesiredConfigByType(KERBEROS_ENV);
if (kerberosEnvConfig == null) {
LOG.debug("Calculating the active identities for {} is being skipped since the kerberos-env configuration is not available",
- clusterName, cluster.getSecurityType().name(), SecurityType.KERBEROS.name());
+ clusterName, cluster.getSecurityType().name(), SecurityType.KERBEROS.name());
} else {
Collection<String> hosts;
String ambariServerHostname = StageUtils.getHostName();
@@ -1741,10 +1768,10 @@ public class KerberosHelperImpl implements KerberosHelper {
// Calculate the current host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
Map<String, Map<String, String>> configurations = calculateConfigurations(cluster,
- hostname,
- kerberosDescriptor,
- false,
- false);
+ hostname,
+ kerberosDescriptor,
+ false,
+ false);
// Create the context to use for filtering Kerberos Identities based on the state of the cluster
Map<String, Object> filterContext = new HashMap<>();
@@ -1754,7 +1781,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, KerberosIdentityDescriptor> hostActiveIdentities = new HashMap<>();
List<KerberosIdentityDescriptor> identities = getActiveIdentities(cluster, hostname,
- serviceName, componentName, kerberosDescriptor, filterContext);
+ serviceName, componentName, kerberosDescriptor, filterContext);
if (hostname.equals(ambariServerHostname)) {
// Determine if we should _calculate_ the Ambari service identities.
@@ -1799,10 +1826,10 @@ public class KerberosHelperImpl implements KerberosHelper {
}
KerberosPrincipalDescriptor resolvedPrincipalDescriptor =
- new KerberosPrincipalDescriptor(principal,
- principalType,
- variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations),
- variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ new KerberosPrincipalDescriptor(principal,
+ principalType,
+ variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations),
+ variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
KerberosKeytabDescriptor resolvedKeytabDescriptor;
@@ -1810,22 +1837,22 @@ public class KerberosHelperImpl implements KerberosHelper {
resolvedKeytabDescriptor = null;
} else {
resolvedKeytabDescriptor =
- new KerberosKeytabDescriptor(
- keytabFile,
- variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
- keytabDescriptor.isCachable());
+ new KerberosKeytabDescriptor(
+ keytabFile,
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
+ keytabDescriptor.isCachable());
}
hostActiveIdentities.put(uniqueKey, new KerberosIdentityDescriptor(
- identity.getName(),
- identity.getReference(),
- resolvedPrincipalDescriptor,
- resolvedKeytabDescriptor,
- identity.getWhen()));
+ identity.getName(),
+ identity.getReference(),
+ resolvedPrincipalDescriptor,
+ resolvedKeytabDescriptor,
+ identity.getWhen()));
}
}
}
@@ -1844,10 +1871,10 @@ public class KerberosHelperImpl implements KerberosHelper {
public List<KerberosIdentityDescriptor> getAmbariServerIdentities(KerberosDescriptor kerberosDescriptor) throws AmbariException {
List<KerberosIdentityDescriptor> ambariIdentities = new ArrayList<>();
- KerberosServiceDescriptor ambariKerberosDescriptor = kerberosDescriptor.getService("AMBARI");
+ KerberosServiceDescriptor ambariKerberosDescriptor = kerberosDescriptor.getService(RootService.AMBARI.name());
if (ambariKerberosDescriptor != null) {
List<KerberosIdentityDescriptor> serviceIdentities = ambariKerberosDescriptor.getIdentities(true, null);
- KerberosComponentDescriptor ambariServerKerberosComponentDescriptor = ambariKerberosDescriptor.getComponent("AMBARI_SERVER");
+ KerberosComponentDescriptor ambariServerKerberosComponentDescriptor = ambariKerberosDescriptor.getComponent(RootComponent.AMBARI_SERVER.name());
if (serviceIdentities != null) {
ambariIdentities.addAll(serviceIdentities);
@@ -1892,44 +1919,42 @@ public class KerberosHelperImpl implements KerberosHelper {
/**
* Creates and saves underlying {@link org.apache.ambari.server.orm.entities.KerberosPrincipalEntity},
- * {@link org.apache.ambari.server.orm.entities.KerberosKeytabEntity} and
- * {@link org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity} entities in JPA storage.
+ * {@link org.apache.ambari.server.orm.entities.KerberosKeytabEntity} entities in JPA storage.
*
* @param resolvedKerberosKeytab kerberos keytab to be persisted
*/
@Override
- public void processResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab) {
+ public void createResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab) {
if (kerberosKeytabDAO.find(resolvedKerberosKeytab.getFile()) == null) {
- kerberosKeytabDAO.create(resolvedKerberosKeytab.getFile());
+ KerberosKeytabEntity kke = new KerberosKeytabEntity(resolvedKerberosKeytab.getFile());
+ kke.setAmbariServerKeytab(resolvedKerberosKeytab.isAmbariServerKeytab());
+ kke.setWriteAmbariJaasFile(resolvedKerberosKeytab.isMustWriteAmbariJaasFile());
+ kke.setOwnerName(resolvedKerberosKeytab.getOwnerName());
+ kke.setOwnerAccess(resolvedKerberosKeytab.getOwnerAccess());
+ kke.setGroupName(resolvedKerberosKeytab.getGroupName());
+ kke.setGroupAccess(resolvedKerberosKeytab.getGroupAccess());
+ kerberosKeytabDAO.create(kke);
}
- for (Pair<Long, Pair<String, String>> principalPair : resolvedKerberosKeytab.getMappedPrincipals()) {
- Pair<String, String> principal = principalPair.getRight();
- String principalName = principal.getLeft();
- String principalType = principal.getRight();
- Long hostId = principalPair.getLeft();
- if (!kerberosPrincipalDAO.exists(principalName)) {
- kerberosPrincipalDAO.create(principalName, "service".equalsIgnoreCase(principalType));
+ for (ResolvedKerberosPrincipal principal : resolvedKerberosKeytab.getPrincipals()) {
+ if (!kerberosPrincipalDAO.exists(principal.getPrincipal())) {
+ kerberosPrincipalDAO.create(principal.getPrincipal(), principal.isService());
}
- if (hostId != null) {
- if(!kerberosPrincipalHostDAO.exists(principalName, hostId, resolvedKerberosKeytab.getFile())) {
- kerberosPrincipalHostDAO.create(principalName, hostId, resolvedKerberosKeytab.getFile());
+ for (Map.Entry<String, String> mappingEntry : principal.getServiceMapping().entries()) {
+ String serviceName = mappingEntry.getKey();
+ HostEntity hostEntity = principal.getHostId() != null ? hostDAO.findById(principal.getHostId()) : null;
+ KerberosKeytabEntity kke = kerberosKeytabDAO.find(resolvedKerberosKeytab.getFile());
+
+ KerberosKeytabPrincipalEntity kkp = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostEntity, kerberosPrincipalDAO.find(principal.getPrincipal()));
+ if(kkp.putServiceMapping(serviceName, mappingEntry.getValue())) {
+ kerberosKeytabPrincipalDAO.merge(kkp);
}
+ kerberosKeytabDAO.merge(kke);
}
}
}
@Override
public void removeStaleKeytabs(Collection<ResolvedKerberosKeytab> expectedKeytabs) {
- List<KerberosKeytabEntity> allKeytabs = kerberosKeytabDAO.findAll();
- Set<KerberosKeytabEntity> staleKeytabs;
- staleKeytabs = allKeytabs != null ? new HashSet<>(allKeytabs) : Collections.emptySet();
- for (ResolvedKerberosKeytab keytab : expectedKeytabs) {
- staleKeytabs.remove(new KerberosKeytabEntity(keytab.getFile()));
- }
- for (KerberosKeytabEntity staleKeytab: staleKeytabs) {
- kerberosPrincipalHostDAO.removeByKeytabPath(staleKeytab.getKeytabPath());
- kerberosKeytabDAO.remove(staleKeytab);
- }
}
@Override
@@ -1951,7 +1976,7 @@ public class KerberosHelperImpl implements KerberosHelper {
}
Set<String> propertyNames = translation.get(configType);
- if(propertyNames == null) {
+ if (propertyNames == null) {
propertyNames = new HashSet<>();
translation.put(configType, propertyNames);
}
@@ -1982,7 +2007,7 @@ public class KerberosHelperImpl implements KerberosHelper {
KerberosPrincipalType expectedType, Map<String, String> kerberosEnvProperties,
KerberosOperationHandler kerberosOperationHandler,
Map<String, Map<String, String>> configurations, String hostname)
- throws AmbariException {
+ throws AmbariException {
Keytab keytab = null;
@@ -2005,12 +2030,12 @@ public class KerberosHelperImpl implements KerberosHelper {
CreatePrincipalsServerAction.CreatePrincipalResult result;
result = injector.getInstance(CreatePrincipalsServerAction.class).createPrincipal(
- principal,
- KerberosPrincipalType.SERVICE.equals(expectedType),
- kerberosEnvProperties,
- kerberosOperationHandler,
- false,
- null);
+ principal,
+ KerberosPrincipalType.SERVICE.equals(expectedType),
+ kerberosEnvProperties,
+ kerberosOperationHandler,
+ false,
+ null);
if (result == null) {
throw new AmbariException("Failed to create the account for " + principal);
@@ -2019,13 +2044,13 @@ public class KerberosHelperImpl implements KerberosHelper {
if (keytabDescriptor != null) {
keytab = injector.getInstance(CreateKeytabFilesServerAction.class).createKeytab(
- principal,
- result.getPassword(),
- result.getKeyNumber(),
- kerberosOperationHandler,
- true,
- true,
- null);
+ principal,
+ result.getPassword(),
+ result.getKeyNumber(),
+ kerberosOperationHandler,
+ true,
+ true,
+ null);
if (keytab == null) {
throw new AmbariException("Failed to create the keytab for " + principal);
@@ -2050,9 +2075,9 @@ public class KerberosHelperImpl implements KerberosHelper {
* @throws AmbariException if any other error occurs while trying to validate the credentials
*/
private void validateKDCCredentials(KerberosDetails kerberosDetails, Cluster cluster) throws KerberosMissingAdminCredentialsException,
- KerberosAdminAuthenticationException,
- KerberosInvalidConfigurationException,
- AmbariException {
+ KerberosAdminAuthenticationException,
+ KerberosInvalidConfigurationException,
+ AmbariException {
if (kerberosDetails == null) {
kerberosDetails = getKerberosDetails(cluster, null);
@@ -2075,34 +2100,34 @@ public class KerberosHelperImpl implements KerberosHelper {
missingCredentials = !operationHandler.testAdministratorCredentials();
} catch (KerberosAdminAuthenticationException e) {
throw new KerberosAdminAuthenticationException(
- "Invalid KDC administrator credentials.\n" +
- "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
- "This may be done by issuing a POST (or PUT for updating) to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
- "{\n" +
- " \"Credential\" : {\n" +
- " \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
- " }\n" +
- "}", e);
+ "Invalid KDC administrator credentials.\n" +
+ "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
+ "This may be done by issuing a POST (or PUT for updating) to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
+ "{\n" +
+ " \"Credential\" : {\n" +
+ " \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
+ " }\n" +
+ "}", e);
} catch (KerberosKDCConnectionException e) {
throw new KerberosInvalidConfigurationException(
- "Failed to connect to KDC - " + e.getMessage() + "\n" +
- "Update the KDC settings in krb5-conf and kerberos-env configurations to correct this issue.",
- e);
+ "Failed to connect to KDC - " + e.getMessage() + "\n" +
+ "Update the KDC settings in krb5-conf and kerberos-env configurations to correct this issue.",
+ e);
} catch (KerberosKDCSSLConnectionException e) {
throw new KerberosInvalidConfigurationException(
- "Failed to connect to KDC - " + e.getMessage() + "\n" +
- "Make sure the server's SSL certificate or CA certificates have been imported into Ambari's truststore.",
- e);
+ "Failed to connect to KDC - " + e.getMessage() + "\n" +
+ "Make sure the server's SSL certificate or CA certificates have been imported into Ambari's truststore.",
+ e);
} catch (KerberosRealmException e) {
throw new KerberosInvalidConfigurationException(
- "Failed to find a KDC for the specified realm - " + e.getMessage() + "\n" +
- "Update the KDC settings in krb5-conf and kerberos-env configurations to correct this issue.",
- e);
+ "Failed to find a KDC for the specified realm - " + e.getMessage() + "\n" +
+ "Update the KDC settings in krb5-conf and kerberos-env configurations to correct this issue.",
+ e);
} catch (KerberosLDAPContainerException e) {
throw new KerberosInvalidConfigurationException(
- "The principal container was not specified\n" +
- "Set the 'container_dn' value in the kerberos-env configuration to correct this issue.",
- e);
+ "The principal container was not specified\n" +
+ "Set the 'container_dn' value in the kerberos-env configuration to correct this issue.",
+ e);
} catch (KerberosOperationException e) {
throw new AmbariException(e.getMessage(), e);
} finally {
@@ -2163,15 +2188,15 @@ public class KerberosHelperImpl implements KerberosHelper {
Set<String> hostsToForceKerberosOperations,
RequestStageContainer requestStageContainer,
final Handler handler)
- throws AmbariException, KerberosOperationException {
+ throws AmbariException, KerberosOperationException {
final KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster, false);
List<ServiceComponentHost> schToProcess = getServiceComponentHostsToProcess(
- cluster,
- kerberosDescriptor,
- serviceComponentFilter,
- hostFilter);
+ cluster,
+ kerberosDescriptor,
+ serviceComponentFilter,
+ hostFilter);
// While iterating over all the ServiceComponentHosts find hosts that have KERBEROS_CLIENT
// components in the INSTALLED state and add them to the hostsWithValidKerberosClient Set.
@@ -2215,29 +2240,29 @@ public class KerberosHelperImpl implements KerberosHelper {
String hostParamsJson = StageUtils.getGson().toJson(hostParams);
String ambariServerHostname = StageUtils.getHostName();
ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(
- "AMBARI_SERVER",
- ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
- System.currentTimeMillis());
+ RootComponent.AMBARI_SERVER.name(),
+ ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
+ System.currentTimeMillis());
RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
// If a RequestStageContainer does not already exist, create a new one...
if (requestStageContainer == null) {
requestStageContainer = new RequestStageContainer(
- actionManager.getNextRequestId(),
- null,
- requestFactory,
- actionManager);
+ actionManager.getNextRequestId(),
+ null,
+ requestFactory,
+ actionManager);
}
// Use the handler implementation to setup the relevant stages.
handler.createStages(cluster, clusterHostInfoJson,
- hostParamsJson, event, roleCommandOrder, kerberosDetails, dataDirectory,
- requestStageContainer, schToProcess, serviceComponentFilter, hostFilter, identityFilter,
- hostsWithValidKerberosClient);
+ hostParamsJson, event, roleCommandOrder, kerberosDetails, dataDirectory,
+ requestStageContainer, schToProcess, serviceComponentFilter, hostFilter, identityFilter,
+ hostsWithValidKerberosClient);
// Add the finalize stage...
handler.addFinalizeOperationStage(cluster, clusterHostInfoJson, hostParamsJson, event,
- dataDirectory, roleCommandOrder, requestStageContainer, kerberosDetails);
+ dataDirectory, roleCommandOrder, requestStageContainer, kerberosDetails);
return requestStageContainer;
}
@@ -2275,7 +2300,6 @@ public class KerberosHelperImpl implements KerberosHelper {
List<ServiceComponentHost> serviceComponentHostsToProcess = new ArrayList<>();
KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster, false);
- KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
// This is needed to help determine which hosts to perform actions for and create tasks for.
Set<String> hostsWithValidKerberosClient = getHostsWithValidKerberosClient(cluster);
@@ -2286,15 +2310,11 @@ public class KerberosHelperImpl implements KerberosHelper {
// this directory until they are distributed to their appropriate hosts.
File dataDirectory = createTemporaryDirectory();
- // Create the file used to store details about principals and keytabs to create
- File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
-
// Calculate the current non-host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
String principal = variableReplacementHelper.replaceVariables("${kerberos-env/service_check_principal_name}@${realm}", configurations);
- String principalType = "user";
String keytabFilePath = variableReplacementHelper.replaceVariables("${keytab_dir}/kerberos.service_check.${short_date}.keytab", configurations);
String keytabFileOwnerName = variableReplacementHelper.replaceVariables("${cluster-env/smokeuser}", configurations);
@@ -2311,7 +2331,6 @@ public class KerberosHelperImpl implements KerberosHelper {
List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name());
if ((serviceComponentHosts != null) && !serviceComponentHosts.isEmpty()) {
- kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
// Iterate over the KERBEROS_CLIENT service component hosts to get the service and
// component-level Kerberos descriptors in order to determine which principals,
@@ -2319,54 +2338,36 @@ public class KerberosHelperImpl implements KerberosHelper {
for (ServiceComponentHost sch : serviceComponentHosts) {
if (sch.getState() == State.INSTALLED) {
String hostname = sch.getHostName();
-
- if(kerberosKeytabDAO.find(keytabFilePath) == null) {
- kerberosKeytabDAO.create(keytabFilePath);
+ KerberosKeytabEntity kke = kerberosKeytabDAO.find(keytabFilePath);
+
+ if (kke == null) {
+ kke = new KerberosKeytabEntity();
+ kke.setKeytabPath(keytabFilePath);
+ kke.setOwnerName(keytabFileOwnerName);
+ kke.setOwnerAccess(keytabFileOwnerAccess);
+ kke.setGroupName(keytabFileGroupName);
+ kke.setGroupAccess(keytabFileGroupAccess);
+ kerberosKeytabDAO.create(kke);
}
// create principals
if (!kerberosPrincipalDAO.exists(principal)) {
kerberosPrincipalDAO.create(principal, false);
}
- if (!kerberosPrincipalHostDAO.exists(principal, sch.getHost().getHostId(), keytabFilePath)) {
- kerberosPrincipalHostDAO.create(principal, sch.getHost().getHostId(), keytabFilePath);
+ KerberosKeytabPrincipalEntity kkp = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostDAO.findById(sch.getHost().getHostId()), kerberosPrincipalDAO.find(principal));
+ if(kkp.putServiceMapping(sch.getServiceName(), sch.getServiceComponentName())) {
+ kerberosKeytabPrincipalDAO.merge(kkp);
}
-
- kerberosIdentityDataFileWriter.writeRecord(
- hostname,
- Service.Type.KERBEROS.name(),
- Role.KERBEROS_CLIENT.name(),
- principal,
- principalType,
- keytabFilePath,
- keytabFileOwnerName,
- keytabFileOwnerAccess,
- keytabFileGroupName,
- keytabFileGroupAccess,
- "false");
-
+ kerberosKeytabDAO.merge(kke);
hostsWithValidKerberosClient.add(hostname);
serviceComponentHostsToProcess.add(sch);
}
}
}
- } catch (IOException e) {
- String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
- LOG.error(message);
- throw new AmbariException(message, e);
} catch (Exception e) {
// make sure to log what is going wrong
LOG.error("Failed " + e);
throw e;
- } finally {
- if (kerberosIdentityDataFileWriter != null) {
- // Make sure the data file is closed
- try {
- kerberosIdentityDataFileWriter.close();
- } catch (IOException e) {
- LOG.warn("Failed to close the index file writer", e);
- }
- }
}
// If there are ServiceComponentHosts to process, make sure the administrator credential
@@ -2380,7 +2381,7 @@ public class KerberosHelperImpl implements KerberosHelper {
FileUtils.deleteDirectory(dataDirectory);
} catch (Throwable t) {
LOG.warn(String.format("The data directory (%s) was not deleted due to an error condition - {%s}",
- dataDirectory.getAbsolutePath(), t.getMessage()), t);
+ dataDirectory.getAbsolutePath(), t.getMessage()), t);
}
throw e;
@@ -2398,31 +2399,31 @@ public class KerberosHelperImpl implements KerberosHelper {
String hostParamsJson = StageUtils.getGson().toJson(hostParams);
String ambariServerHostname = StageUtils.getHostName();
ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(
- "AMBARI_SERVER",
- ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
- System.currentTimeMillis());
+ RootComponent.AMBARI_SERVER.name(),
+ ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
+ System.currentTimeMillis());
RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
-
// If a RequestStageContainer does not already exist, create a new one...
if (requestStageContainer == null) {
requestStageContainer = new RequestStageContainer(
- actionManager.getNextRequestId(),
- null,
- requestFactory,
- actionManager);
+ actionManager.getNextRequestId(),
+ null,
+ requestFactory,
+ actionManager);
}
// Use the handler implementation to setup the relevant stages.
// Set the service/component filter to an empty map since the service/component processing
// was done above.
handler.createStages(cluster,
- clusterHostInfoJson, hostParamsJson, event, roleCommandOrder, kerberosDetails,
- dataDirectory, requestStageContainer, serviceComponentHostsToProcess,
- Collections.emptyMap(), null, null, hostsWithValidKerberosClient);
+ clusterHostInfoJson, hostParamsJson, event, roleCommandOrder, kerberosDetails,
+ dataDirectory, requestStageContainer, serviceComponentHostsToProcess,
+ Collections.singletonMap("KERBEROS", Lists.newArrayList("KERBEROS_CLIENT")),
+ null, Sets.newHashSet(principal), hostsWithValidKerberosClient);
handler.addFinalizeOperationStage(cluster, clusterHostInfoJson, hostParamsJson, event,
- dataDirectory, roleCommandOrder, requestStageContainer, kerberosDetails);
+ dataDirectory, roleCommandOrder, requestStageContainer, kerberosDetails);
}
return requestStageContainer;
@@ -2441,7 +2442,7 @@ public class KerberosHelperImpl implements KerberosHelper {
* @throws AmbariException
*/
private KerberosDetails getKerberosDetails(Cluster cluster, Boolean manageIdentities)
- throws KerberosInvalidConfigurationException, AmbariException {
+ throws KerberosInvalidConfigurationException, AmbariException {
KerberosDetails kerberosDetails = new KerberosDetails();
@@ -2527,7 +2528,7 @@ public class KerberosHelperImpl implements KerberosHelper {
do {
directory = new File(temporaryDirectory, String.format("%s%d-%d.d",
- KerberosServerAction.DATA_DIRECTORY_PREFIX, now, tries));
+ KerberosServerAction.DATA_DIRECTORY_PREFIX, now, tries));
if ((directory.exists()) || !directory.mkdirs()) {
directory = null; // Rest and try again...
@@ -2603,8 +2604,8 @@ public class KerberosHelperImpl implements KerberosHelper {
for (Map.Entry<String, String> property : updates.entrySet()) {
existingProperties.put(
- variableReplacementHelper.replaceVariables(property.getKey(), replacements),
- variableReplacementHelper.replaceVariables(property.getValue(), replacements)
+ variableReplacementHelper.replaceVariables(property.getKey(), replacements),
+ variableReplacementHelper.replaceVariables(property.getValue(), replacements)
);
}
}
@@ -2632,8 +2633,8 @@ public class KerberosHelperImpl implements KerberosHelper {
KerberosPrincipalDescriptor principalDescriptor = identity.getPrincipalDescriptor();
if (principalDescriptor != null) {
authToLocalBuilder.addRule(
- variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations),
- variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations),
+ variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
}
}
}
@@ -2693,12 +2694,12 @@ public class KerberosHelperImpl implements KerberosHelper {
String requestContext, String commandParams, String hostParams) {
Stage stage = stageFactory.createNew(requestId,
- BASE_LOG_DIR + File.pathSeparator + requestId,
- cluster.getClusterName(),
- cluster.getClusterId(),
- requestContext,
- commandParams,
- hostParams);
+ BASE_LOG_DIR + File.pathSeparator + requestId,
+ cluster.getClusterName(),
+ cluster.getClusterId(),
+ requestContext,
+ commandParams,
+ hostParams);
stage.setStageId(id);
return stage;
@@ -2732,9 +2733,9 @@ public class KerberosHelperImpl implements KerberosHelper {
Stage stage = createNewStage(id, cluster, requestId, requestContext, commandParams, hostParams);
stage.addServerActionCommand(actionClass.getName(), null, Role.AMBARI_SERVER_ACTION,
- RoleCommand.EXECUTE, cluster.getClusterName(), event, commandParameters, commandDetail,
- ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), timeout,
- false, false);
+ RoleCommand.EXECUTE, cluster.getClusterName(), event, commandParameters, commandDetail,
+ ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), timeout,
+ false, false);
return stage;
}
@@ -2748,7 +2749,7 @@ public class KerberosHelperImpl implements KerberosHelper {
* @throws org.apache.ambari.server.AmbariException
*/
private List<String> createUniqueHostList(Collection<ServiceComponentHost> serviceComponentHosts, Set<HostState> allowedStates)
- throws AmbariException {
+ throws AmbariException {
Set<String> hostNames = new HashSet<>();
Set<String> visitedHostNames = new HashSet<>();
@@ -2785,7 +2786,7 @@ public class KerberosHelperImpl implements KerberosHelper {
public boolean shouldExecuteCustomOperations(SecurityType requestSecurityType, Map<String, String> requestProperties) {
if (((requestSecurityType == SecurityType.KERBEROS) || (requestSecurityType == SecurityType.NONE)) &&
- (requestProperties != null) && !requestProperties.isEmpty()) {
+ (requestProperties != null) && !requestProperties.isEmpty()) {
for (SupportedCustomOperation type : SupportedCustomOperation.values()) {
if (requestProperties.containsKey(type.name().toLowerCase())) {
return true;
@@ -2800,8 +2801,8 @@ public class KerberosHelperImpl implements KerberosHelper {
String value = (requestProperties == null) ? null : requestProperties.get(DIRECTIVE_MANAGE_KERBEROS_IDENTITIES);
return (value == null)
- ? null
- : !"false".equalsIgnoreCase(value);
+ ? null
+ : !"false".equalsIgnoreCase(value);
}
@Override
@@ -2878,7 +2879,7 @@ public class KerberosHelperImpl implements KerberosHelper {
String componentName,
KerberosDescriptor kerberosDescriptor,
Map<String, Object> filterContext)
- throws AmbariException {
+ throws AmbariException {
List<KerberosIdentityDescriptor> identities = new ArrayList<>();
@@ -2890,7 +2891,7 @@ public class KerberosHelperImpl implements KerberosHelper {
String schComponentName = serviceComponentHost.getServiceComponentName();
if (((serviceName == null) || serviceName.equals(schServiceName)) &&
- ((componentName == null) || componentName.equals(schComponentName))) {
+ ((componentName == null) || componentName.equals(schComponentName))) {
KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(schServiceName);
@@ -2968,7 +2969,7 @@ public class KerberosHelperImpl implements KerberosHelper {
*/
private Map<String, Map<String, String>> addAdditionalConfigurations(Cluster cluster, Map<String, Map<String, String>> configurations,
String hostname, Map<String, String> kerberosDescriptorProperties)
- throws AmbariException {
+ throws AmbariException {
// A map to hold un-categorized properties. This may come from the KerberosDescriptor
// and will also contain a value for the current host
@@ -3140,7 +3141,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Map<String, String>> kerberosConfigurations,
Map<String, Map<String, String>> configurations,
Map<String, Set<String>> propertiesToIgnore)
- throws AmbariException {
+ throws AmbariException {
if (identityConfigurations != null) {
for (Map.Entry<String, Map<String, String>> identitiyEntry : identityConfigurations.entrySet()) {
String configType = identitiyEntry.getKey();
@@ -3185,7 +3186,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Cluster cluster,
KerberosDescriptor kerberosDescriptor,
boolean calculateClusterHostInfo)
- throws AmbariException {
+ throws AmbariException {
Map<String, KerberosServiceDescriptor> serviceDescriptorMap = kerberosDescriptor.getServices();
@@ -3355,25 +3356,25 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, ? extends Collection<String>> serviceComponentFilter,
Set<String> hostFilter, Collection<String> identityFilter,
Set<String> hostsWithValidKerberosClient)
- throws AmbariException;
+ throws AmbariException;
public void addPrepareEnableKerberosOperationsStage(Cluster cluster, String clusterHostInfoJson,
String hostParamsJson, ServiceComponentHostServerActionEvent event,
Map<String, String> commandParameters,
RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer)
- throws AmbariException {
+ throws AmbariException {
Stage stage = createServerActionStage(requestStageContainer.getLastStageId(),
- cluster,
- requestStageContainer.getId(),
- "Preparing Operations",
- "{}",
- hostParamsJson,
- PrepareEnableKerberosServerAction.class,
- event,
- commandParameters,
- "Preparing Operations",
- configuration.getDefaultServerTaskTimeout());
+ cluster,
+ requestStageContainer.getId(),
+ "Preparing Operations",
+ "{}",
+ hostParamsJson,
+ PrepareEnableKerberosServerAction.class,
+ event,
+ commandParameters,
+ "Preparing Operations",
+ configuration.getDefaultServerTaskTimeout());
RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
roleGraph.build(stage);
@@ -3386,18 +3387,18 @@ public class KerberosHelperImpl implements KerberosHelper {
String hostParamsJson, ServiceComponentHostServerActionEvent event,
Map<String, String> commandParameters,
RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer)
- throws AmbariException {
+ throws AmbariException {
Stage stage = createServerActionStage(requestStageContainer.getLastStageId(),
- cluster,
- requestStageContainer.getId(),
- "Preparing Operations",
- "{}",
- hostParamsJson,
- PrepareKerberosIdentitiesServerAction.class,
- event,
- commandParameters,
- "Preparing Operations",
- configuration.getDefaultServerTaskTimeout());
+ cluster,
+ requestStageContainer.getId(),
+ "Preparing Operations",
+ "{}",
+ hostParamsJson,
+ PrepareKerberosIdentitiesServerAction.class,
+ event,
+ commandParameters,
+ "Preparing Operations",
+ configuration.getDefaultServerTaskTimeout());
RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
roleGraph.build(stage);
@@ -3410,18 +3411,18 @@ public class KerberosHelperImpl implements KerberosHelper {
String hostParamsJson, ServiceComponentHostServerActionEvent event,
Map<String, String> commandParameters,
RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer)
- throws AmbariException {
+ throws AmbariException {
Stage stage = createServerActionStage(requestStageContainer.getLastStageId(),
- cluster,
- requestStageContainer.getId(),
- "Preparing Operations",
- "{}",
- hostParamsJson,
- PrepareDisableKerberosServerAction.class,
- event,
- commandParameters,
- "Preparing Operations",
- configuration.getDefaultServerTaskTimeout());
+ cluster,
+ requestStageContainer.getId(),
+ "Preparing Operations",
+ "{}",
+ hostParamsJson,
+ PrepareDisableKerberosServerAction.class,
+ event,
+ commandParameters,
+ "Preparing Operations",
+ configuration.getDefaultServerTaskTimeout());
RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
roleGraph.build(stage);
@@ -3434,18 +3435,18 @@ public class KerberosHelperImpl implements KerberosHelper {
String hostParamsJson, ServiceComponentHostServerActionEvent event,
Map<String, String> commandParameters,
RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer)
- throws AmbariException {
+ throws AmbariException {
Stage stage = createServerActionStage(requestStageContainer.getLastStageId(),
- cluster,
- requestStageContainer.getId(),
- "Create Principals",
- "{}",
- hostParamsJson,
- CreatePrincipalsServerAction.class,
- event,
- commandParameters,
- "Create Principals",
- Math.max(ServerAction.DEFAULT_LONG_RUNNING_TASK_TIMEOUT_SECONDS, configuration.getDefaultServerTaskTimeout()));
+ cluster,
+ requestStageContainer.getId(),
+ "Create Principals",
+ "{}",
+ hostParamsJson,
+ CreatePrincipalsServerAction.class,
+ event,
+ commandParameters,
+ "Create Principals",
+ Math.max(ServerAction.DEFAULT_LONG_RUNNING_TASK_TIMEOUT_SECONDS, configuration.getDefaultServerTaskTimeout()));
RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
roleGraph.build(stage);
@@ -3458,18 +3459,18 @@ public class KerberosHelperImpl implements KerberosHelper {
<TRUNCATED>
[14/51] [abbrv] ambari git commit: AMBARI-22644 - Node Managers fail
to start after Spark2 is patched due to CNF YarnShuffleService
(jonathanhurley)
Posted by am...@apache.org.
AMBARI-22644 - Node Managers fail to start after Spark2 is patched due to CNF YarnShuffleService (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d87b21c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d87b21c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d87b21c
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 1d87b21cf34c51bc19f80199e2b8641474fe098a
Parents: eab6722
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Dec 13 12:33:26 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Dec 18 15:10:43 2017 -0500
----------------------------------------------------------------------
.../2.1.0.2.0/package/scripts/params_linux.py | 19 ++++++++++++-------
.../3.0.0.3.0/package/scripts/params_linux.py | 20 ++++++++++++--------
.../services/YARN/configuration/yarn-site.xml | 4 ++--
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 2 --
.../services/YARN/configuration/yarn-site.xml | 4 ++--
5 files changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d87b21c/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 9997779..4a49822 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -89,16 +89,20 @@ def get_spark_version(service_name, component_name, yarn_version):
"""
Attempts to calculate the correct version placeholder value for spark or spark2 based on
what is installed in the cluster. If Spark is not installed, then this value will need to be
- that of YARN so it can still find the correct shuffle class.
+ that of YARN so it can still find the correct spark class.
On cluster installs, we have not yet calcualted any versions and all known values could be None.
This doesn't affect daemons, but it does affect client-only hosts where they will never receive
a start command after install. Therefore, this function will attempt to use stack-select as a
last resort to get a value value.
+
+ ATS needs this since it relies on packages installed by Spark. Some classes, like the shuffle
+ classes, are not provided by spark, but by a dependent RPM to YARN, so they do not use this
+ value.
:param service_name: the service name (SPARK, SPARK2, etc)
:param component_name: the component name (SPARK_CLIENT, etc)
:param yarn_version: the default version of Yarn to use if no spark is installed
- :return: a value for the version placeholder in shuffle classpath properties
+ :return: a value for the version placeholder in spark classpath properties
"""
# start off seeing if we need to populate a default value for YARN
if yarn_version is None:
@@ -106,18 +110,19 @@ def get_spark_version(service_name, component_name, yarn_version):
component_name = "YARN_CLIENT")
# now try to get the version of spark/spark2, defaulting to the version if YARN
- shuffle_classpath_version = component_version.get_component_repository_version(service_name = service_name,
+ spark_classpath_version = component_version.get_component_repository_version(service_name = service_name,
component_name = component_name, default_value = yarn_version)
# even with the default of using YARN's version, on an install this might be None since we haven't
# calculated the version of YARN yet - use stack_select as a last ditch effort
- if shuffle_classpath_version is None:
+ if spark_classpath_version is None:
try:
- shuffle_classpath_version = stack_select.get_role_component_current_stack_version()
+ spark_classpath_version = stack_select.get_role_component_current_stack_version()
except:
- Logger.exception("Unable to query for the correct shuffle classpath")
+ Logger.exception("Unable to query for the correct spark version to use when building classpaths")
+
+ return spark_classpath_version
- return shuffle_classpath_version
# these are used to render the classpath for picking up Spark classes
# in the event that spark is not installed, then we must default to the vesrion of YARN installed
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d87b21c/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
index 8e9632c..9afd112 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
@@ -89,16 +89,20 @@ def get_spark_version(service_name, component_name, yarn_version):
"""
Attempts to calculate the correct version placeholder value for spark or spark2 based on
what is installed in the cluster. If Spark is not installed, then this value will need to be
- that of YARN so it can still find the correct shuffle class.
+ that of YARN so it can still find the correct spark class.
On cluster installs, we have not yet calcualted any versions and all known values could be None.
This doesn't affect daemons, but it does affect client-only hosts where they will never receive
a start command after install. Therefore, this function will attempt to use stack-select as a
last resort to get a value value.
+
+ ATS needs this since it relies on packages installed by Spark. Some classes, like the shuffle
+ classes, are not provided by spark, but by a dependent RPM to YARN, so they do not use this
+ value.
:param service_name: the service name (SPARK, SPARK2, etc)
:param component_name: the component name (SPARK_CLIENT, etc)
:param yarn_version: the default version of Yarn to use if no spark is installed
- :return: a value for the version placeholder in shuffle classpath properties
+ :return: a value for the version placeholder in spark classpath properties
"""
# start off seeing if we need to populate a default value for YARN
if yarn_version is None:
@@ -106,18 +110,19 @@ def get_spark_version(service_name, component_name, yarn_version):
component_name = "YARN_CLIENT")
# now try to get the version of spark/spark2, defaulting to the version if YARN
- shuffle_classpath_version = component_version.get_component_repository_version(service_name = service_name,
+ spark_classpath_version = component_version.get_component_repository_version(service_name = service_name,
component_name = component_name, default_value = yarn_version)
# even with the default of using YARN's version, on an install this might be None since we haven't
# calculated the version of YARN yet - use stack_select as a last ditch effort
- if shuffle_classpath_version is None:
+ if spark_classpath_version is None:
try:
- shuffle_classpath_version = stack_select.get_role_component_current_stack_version()
+ spark_classpath_version = stack_select.get_role_component_current_stack_version()
except:
- Logger.exception("Unable to query for the correct shuffle classpath")
+ Logger.exception("Unable to query for the correct spark version to use when building classpaths")
+
+ return spark_classpath_version
- return shuffle_classpath_version
# these are used to render the classpath for picking up Spark classes
# in the event that spark is not installed, then we must default to the vesrion of YARN installed
@@ -317,7 +322,6 @@ if include_file_path and manage_include_files:
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
has_ats = not len(ats_host) == 0
-
# don't using len(nm_hosts) here, because check can take too much time on large clusters
number_of_nm = 1
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d87b21c/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
index 29833fb..b6fadcb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -31,13 +31,13 @@
</property>
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
- <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
+ <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
<description>The auxiliary service classpath to use for Spark</description>
<on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
- <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
+ <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
<description>The auxiliary service classpath to use for Spark 2</description>
<on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d87b21c/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 9c3d0dd..d8f71cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -183,8 +183,6 @@
<changes>
<definition xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths">
<type>yarn-site</type>
- <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/{{spark_version}}/spark/aux/*"/>
- <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/{{spark2_version}}/spark2/aux/*"/>
<replace key="yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" find="${hdp.version}/spark/" replace-with="{{spark_version}}/spark/"/>
</definition>
</changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d87b21c/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
index d32e2de..01551c4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
@@ -20,7 +20,7 @@
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
- <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
+ <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
<description>The auxiliary service classpath to use for Spark</description>
<on-ambari-upgrade add="false"/>
</property>
@@ -41,7 +41,7 @@
<!-- These configs were inherited from HDP 2.5 -->
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
- <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
+ <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
<description>The auxiliary service classpath to use for Spark 2</description>
<on-ambari-upgrade add="false"/>
</property>
[48/51] [abbrv] ambari git commit: AMBARI-22449. Improved
service/component dependency support (amagyar)
Posted by am...@apache.org.
AMBARI-22449. Improved service/component dependency support (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a982d2e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a982d2e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a982d2e
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 7a982d2edce853b5b16395249c8da251ad147572
Parents: 683574c
Author: Attila Magyar <am...@hortonworks.com>
Authored: Thu Nov 16 15:35:41 2017 +0100
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Fri Dec 22 09:57:59 2017 +0100
----------------------------------------------------------------------
.../app/controllers/wizard/step4_controller.js | 4 +--
ambari-web/app/messages.js | 3 +-
ambari-web/app/models/stack_service.js | 30 +++++++++++---------
3 files changed, 21 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a982d2e/ambari-web/app/controllers/wizard/step4_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step4_controller.js b/ambari-web/app/controllers/wizard/step4_controller.js
index d39164b..8a62f3b 100644
--- a/ambari-web/app/controllers/wizard/step4_controller.js
+++ b/ambari-web/app/controllers/wizard/step4_controller.js
@@ -480,11 +480,11 @@ App.WizardStep4Controller = Em.ArrayController.extend({
needToAddMissingDependency: function (missingDependency, i18nSuffix, callback, id) {
var self = this;
var displayName = missingDependency.get('displayName');
- if (missingDependency.hasMultipleOptions()) {
+ if (missingDependency.get('hasMultipleOptions')) {
return this.showDependencyPopup(
id,
Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.header').format(displayName),
- Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.body2').format(displayName, missingDependency.displayOptions()),
+ Em.I18n.t('installer.step4.' + i18nSuffix + '.popup.body.multiOptions').format(displayName, missingDependency.get('displayOptions')),
callback
);
} else {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a982d2e/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index f498a58..d002aa0 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -883,7 +883,8 @@ Em.I18n.translations = {
'installer.step4.multipleDFS.popup.body':'You selected more than one file system. We will automatically select only {0}. Is this OK?',
'installer.step4.serviceCheck.popup.header':'{0} Needed',
'installer.step4.serviceCheck.popup.body':'You did not select {0}, but it is needed by other services you selected. We will automatically add {1}. Is this OK?',
- 'installer.step4.serviceCheck.popup.body2':'You did not select {0}, but it is needed by other services you selected. Select a compatible service from the following list: {1}',
+ 'installer.step4.serviceCheck.popup.body.multiOptions':'You did not select {0}, but it is needed by other services you selected. Select a compatible service from the following list: {1}',
+ 'installer.step4.hcfs.displayName':'a Hadoop Compatible File System',
'installer.step4.limitedFunctionality.popup.header':'Limited Functionality Warning',
'installer.step4.ambariMetricsCheck.popup.body':'Ambari Metrics collects metrics from the cluster and makes them available to Ambari. If you do not install Ambari Metrics service, metrics will not be accessible from Ambari. Are you sure you want to proceed without Ambari Metrics?',
'installer.step4.ambariInfraCheck.popup.body':'Since Ambari Infra is not selected, you must supply your own Solr to make Atlas work. Are you sure you want to proceed?',
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a982d2e/ambari-web/app/models/stack_service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js
index 46dfee5..a84816e 100644
--- a/ambari-web/app/models/stack_service.js
+++ b/ambari-web/app/models/stack_service.js
@@ -23,11 +23,11 @@ require('models/configs/objects/service_config_category');
var Dependency = Ember.Object.extend({
name: Ember.computed('service', function() {
- return this.get('service').get('serviceName');
+ return this.get('service.serviceName');
}),
displayName: Ember.computed('service', function() {
- return this.get('service').get('displayNameOnSelectServicePage');
+ return this.get('service.displayNameOnSelectServicePage');
}),
compatibleServices: function(services) {
@@ -40,8 +40,8 @@ var Dependency = Ember.Object.extend({
});
var HcfsDependency = Dependency.extend({
- displayName: Ember.computed('service', function() {
- return 'a Hadoop Compatible File System';
+ displayName: Ember.computed(function() {
+ return Em.I18n.t('installer.step4.hcfs.displayName');
}),
compatibleServices: function(services) {
@@ -58,17 +58,17 @@ Dependency.reopenClass({
});
var MissingDependency = Ember.Object.extend({
- hasMultipleOptions: function() {
+ hasMultipleOptions: Ember.computed('compatibleServices', function() {
return this.get('compatibleServices').length > 1;
- },
+ }),
selectFirstCompatible: function() {
this.get('compatibleServices')[0].set('isSelected', true);
},
- displayOptions: function() {
+ displayOptions: Ember.computed('compatibleServices', function() {
return this.get('compatibleServices').mapProperty('serviceName').join(', ');
- }
+ })
});
/**
@@ -122,10 +122,14 @@ App.StackService = DS.Model.extend({
dependentServiceNames: DS.attr('array', {defaultValue: []}),
dependencies: function(availableServices) {
- return this.get('requiredServices')
- .map(function (serviceName) { return availableServices.findProperty('serviceName', serviceName)})
- .filter(function (each) { return !!each })
- .map(function (service) { return Dependency.fromService(service); });
+ var result = [];
+ this.get('requiredServices').forEach(function(serviceName) {
+ var service = availableServices.findProperty('serviceName', serviceName);
+ if (service) {
+ result.push(Dependency.fromService(service));
+ }
+ });
+ return result;
},
/**
@@ -144,7 +148,7 @@ App.StackService = DS.Model.extend({
},
_addMissingDependency: function(dependency, availableServices, missingDependencies) {
- if(!missingDependencies.some(function(each) { return each.get('serviceName') == dependency.get('name'); })) {
+ if(!missingDependencies.someProperty('serviceName', dependency.get('name'))) {
missingDependencies.push(MissingDependency.create({
serviceName: dependency.get('name'),
displayName: dependency.get('displayName'),
[27/51] [abbrv] ambari git commit: AMBARI-22682 Ambari 3.0 Admin
View: Add visual-search box to all pages. (atkach)
Posted by am...@apache.org.
AMBARI-22682 Ambari 3.0 Admin View: Add visual-search box to all pages. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/81c04545
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/81c04545
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/81c04545
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: 81c045452e37adc6e68379a125026ac98f8ed103
Parents: 9ea3fa1
Author: Andrii Tkach <at...@apache.org>
Authored: Thu Dec 21 15:24:55 2017 +0200
Committer: Andrii Tkach <at...@apache.org>
Committed: Thu Dec 21 15:24:55 2017 +0200
----------------------------------------------------------------------
.../main/resources/ui/admin-web/app/index.html | 2 +
.../app/scripts/controllers/SideNavCtrl.js | 6 +-
.../controllers/ambariViews/ViewsListCtrl.js | 184 +++++------
.../remoteClusters/RemoteClustersListCtrl.js | 109 +++----
.../stackVersions/StackVersionsEditCtrl.js | 2 +-
.../stackVersions/StackVersionsListCtrl.js | 191 +++++-------
.../userManagement/GroupsListCtrl.js | 86 ++----
.../controllers/userManagement/UsersListCtrl.js | 132 ++++----
.../app/scripts/directives/comboSearch.js | 39 ++-
.../admin-web/app/scripts/services/Cluster.js | 3 +-
.../admin-web/app/scripts/services/Filters.js | 84 +++++
.../ui/admin-web/app/scripts/services/Group.js | 11 +-
.../app/scripts/services/Pagination.js | 59 ++++
.../app/scripts/services/RemoteCluster.js | 10 +-
.../ui/admin-web/app/scripts/services/Stack.js | 32 +-
.../ui/admin-web/app/scripts/services/User.js | 13 +-
.../ui/admin-web/app/styles/user-management.css | 4 +
.../app/views/ambariViews/viewsList.html | 5 +-
.../app/views/remoteClusters/list.html | 33 +-
.../admin-web/app/views/stackVersions/list.html | 34 +--
.../app/views/userManagement/groupsList.html | 51 ++--
.../app/views/userManagement/usersList.html | 61 ++--
.../ambariViews/ViewsListCtrl_test.js | 66 +---
.../userManagement/GroupsListCtrl_test.js | 100 +-----
.../userManagement/UsersListCtrl_test.js | 306 -------------------
.../test/unit/directives/comboSearch_test.js | 13 -
.../test/unit/services/Filters_test.js | 161 ++++++++++
.../test/unit/services/Pagination_test.js | 72 +++++
28 files changed, 805 insertions(+), 1064 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/index.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/index.html b/ambari-admin/src/main/resources/ui/admin-web/app/index.html
index a1346ed..a9c6984 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/index.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/index.html
@@ -170,6 +170,8 @@
<script src="scripts/services/AddRepositoryModal.js"></script>
<script src="scripts/services/AddVersionModal.js"></script>
<script src="scripts/services/RoleDetailsModal.js"></script>
+<script src="scripts/services/Pagination.js"></script>
+<script src="scripts/services/Filters.js"></script>
<!-- endbuild -->
</body>
</html>
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/SideNavCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/SideNavCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/SideNavCtrl.js
index 558d110..6bf356d 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/SideNavCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/SideNavCtrl.js
@@ -29,11 +29,7 @@ angular.module('ambariAdminConsole')
}, true);
function loadRepos() {
- Stack.allRepos({version: '',
- cluster: {
- options: [],
- current: null
- }}, {}).then(function (repos) {
+ Stack.allRepos().then(function (repos) {
$scope.totalRepos = repos.itemTotal;
});
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
index 8c61a25..f536e50 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js
@@ -18,7 +18,9 @@
'use strict';
angular.module('ambariAdminConsole')
-.controller('ViewsListCtrl',['$scope', 'View','$modal', 'Alert', 'ConfirmationModal', '$translate', 'Settings', function($scope, View, $modal, Alert, ConfirmationModal, $translate, Settings) {
+.controller('ViewsListCtrl',
+['$scope', 'View','$modal', 'Alert', 'ConfirmationModal', '$translate', 'Settings', 'Pagination', 'Filters',
+function($scope, View, $modal, Alert, ConfirmationModal, $translate, Settings, Pagination, Filters) {
var $t = $translate.instant;
var VIEWS_VERSION_STATUS_TIMEOUT = 5000;
$scope.isLoading = false;
@@ -33,6 +35,9 @@ angular.module('ambariAdminConsole')
{
key: 'url',
label: $t('urls.url'),
+ customValueConverter: function(item) {
+ return '/main/view/' + item.view_name + '/' + item.short_url;
+ },
options: []
},
{
@@ -46,136 +51,32 @@ angular.module('ambariAdminConsole')
options: []
}
];
-
- function checkViewVersionStatus(view, versionObj, versionNumber) {
- var deferred = View.checkViewVersionStatus(view.view_name, versionNumber);
-
- deferred.promise.then(function (status) {
- if (versionNeedStatusUpdate(status)) {
- setTimeout(function() {
- checkViewVersionStatus(view, versionObj, versionNumber);
- }, VIEWS_VERSION_STATUS_TIMEOUT);
- } else {
- versionObj.status = status;
- angular.forEach(view.versions, function (version) {
- if (version.status === 'DEPLOYED') {
- view.canCreateInstance = true;
- }
- })
- }
- });
- }
-
- function versionNeedStatusUpdate(status) {
- return status !== 'DEPLOYED' && status !== 'ERROR';
- }
-
- function loadViews() {
- $scope.isLoading = true;
- View.all().then(function (views) {
- $scope.isLoading = false;
- $scope.views = views;
- $scope.instances = [];
- angular.forEach(views, function (view) {
- angular.forEach(view.versions, function (versionObj, versionNumber) {
- if (versionNeedStatusUpdate(versionObj.status)) {
- checkViewVersionStatus(view, versionObj, versionNumber);
- }
- });
- angular.forEach(view.instances, function (instance) {
- instance.ViewInstanceInfo.short_url_name = instance.ViewInstanceInfo.short_url_name || '';
- instance.ViewInstanceInfo.short_url = instance.ViewInstanceInfo.short_url || '';
- instance.ViewInstanceInfo.versionObj = view.versions[instance.ViewInstanceInfo.version] || {};
- $scope.instances.push(instance.ViewInstanceInfo);
- });
- });
- $scope.initFilterOptions();
- $scope.filterInstances();
- }).catch(function (data) {
- Alert.error($t('views.alerts.cannotLoadViews'), data.data.message);
- });
- }
-
- function showInstancesOnPage() {
- var startIndex = ($scope.currentPage - 1) * $scope.instancesPerPage + 1;
- var endIndex = $scope.currentPage * $scope.instancesPerPage;
- var showedCount = 0;
- var filteredCount = 0;
-
- angular.forEach($scope.instances, function(instance) {
- instance.isShowed = false;
- if (instance.isFiltered) {
- filteredCount++;
- if (filteredCount >= startIndex && filteredCount <= endIndex) {
- instance.isShowed = true;
- showedCount++;
- }
- }
- });
- $scope.tableInfo.showed = showedCount;
- }
-
$scope.views = [];
$scope.instances = [];
- $scope.instancesPerPage = 10;
- $scope.currentPage = 1;
- $scope.maxVisiblePages = 10;
$scope.tableInfo = {
filtered: 0,
- showed: 0
+ showed: 0,
+ total: 0
};
+ $scope.pagination = Pagination.create();
- loadViews();
+ $scope.resetPagination = function() {
+ $scope.pagination.resetPagination($scope.instances, $scope.tableInfo);
+ };
- $scope.initFilterOptions = function() {
- $scope.filters.forEach(function(filter) {
- filter.options = $.unique($scope.instances.map(function(instance) {
- if (filter.key === 'url') {
- return '/main/view/' + instance.view_name + '/' + instance.short_url;
- }
- return instance[filter.key];
- })).map(function(item) {
- return {
- key: item,
- label: item
- }
- });
- });
+ $scope.pageChanged = function() {
+ $scope.pagination.pageChanged($scope.instances, $scope.tableInfo);
};
$scope.filterInstances = function(appliedFilters) {
- var filteredCount = 0;
- angular.forEach($scope.instances, function(instance) {
- instance.isFiltered = !(appliedFilters && appliedFilters.length > 0 && appliedFilters.some(function(filter) {
- if (filter.key === 'url') {
- return filter.values.every(function(value) {
- return ('/main/view/' + instance.view_name + '/' + instance.short_url).indexOf(value) === -1;
- });
- }
- return filter.values.every(function(value) {
- return instance[filter.key].indexOf(value) === -1;
- });
- }));
-
- filteredCount += ~~instance.isFiltered;
- });
- $scope.tableInfo.filtered = filteredCount;
- $scope.resetPagination();
+ $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.instances, $scope.filters);
+ $scope.pagination.resetPagination($scope.instances, $scope.tableInfo);
};
$scope.toggleSearchBox = function() {
$('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
};
- $scope.pageChanged = function() {
- showInstancesOnPage();
- };
-
- $scope.resetPagination = function() {
- $scope.currentPage = 1;
- showInstancesOnPage();
- };
-
$scope.cloneInstance = function(instanceClone) {
$scope.createInstance(instanceClone);
};
@@ -217,4 +118,57 @@ angular.module('ambariAdminConsole')
});
});
};
+
+ loadViews();
+
+ function checkViewVersionStatus(view, versionObj, versionNumber) {
+ var deferred = View.checkViewVersionStatus(view.view_name, versionNumber);
+
+ deferred.promise.then(function (status) {
+ if (versionNeedStatusUpdate(status)) {
+ setTimeout(function() {
+ checkViewVersionStatus(view, versionObj, versionNumber);
+ }, VIEWS_VERSION_STATUS_TIMEOUT);
+ } else {
+ versionObj.status = status;
+ angular.forEach(view.versions, function (version) {
+ if (version.status === 'DEPLOYED') {
+ view.canCreateInstance = true;
+ }
+ })
+ }
+ });
+ }
+
+ function versionNeedStatusUpdate(status) {
+ return status !== 'DEPLOYED' && status !== 'ERROR';
+ }
+
+ function loadViews() {
+ $scope.isLoading = true;
+ View.all().then(function (views) {
+ $scope.isLoading = false;
+ $scope.views = views;
+ $scope.instances = [];
+ angular.forEach(views, function (view) {
+ angular.forEach(view.versions, function (versionObj, versionNumber) {
+ if (versionNeedStatusUpdate(versionObj.status)) {
+ checkViewVersionStatus(view, versionObj, versionNumber);
+ }
+ });
+ angular.forEach(view.instances, function (instance) {
+ instance.ViewInstanceInfo.short_url_name = instance.ViewInstanceInfo.short_url_name || '';
+ instance.ViewInstanceInfo.short_url = instance.ViewInstanceInfo.short_url || '';
+ instance.ViewInstanceInfo.versionObj = view.versions[instance.ViewInstanceInfo.version] || {};
+ $scope.instances.push(instance.ViewInstanceInfo);
+ });
+ });
+ $scope.tableInfo.total = $scope.instances.length;
+ Filters.initFilterOptions($scope.filters, $scope.instances);
+ $scope.filterInstances();
+ }).catch(function (data) {
+ Alert.error($t('views.alerts.cannotLoadViews'), data.data.message);
+ });
+ }
+
}]);
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
index 4726357..5944d20 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js
@@ -18,93 +18,74 @@
'use strict';
angular.module('ambariAdminConsole')
-.controller('RemoteClustersListCtrl', ['$scope', '$routeParams', '$translate', 'RemoteCluster', 'Settings', function ($scope, $routeParams, $translate, RemoteCluster, Settings) {
+.controller('RemoteClustersListCtrl',
+['$scope', '$routeParams', '$translate', 'RemoteCluster', 'Settings', 'Pagination', 'Filters',
+function ($scope, $routeParams, $translate, RemoteCluster, Settings, Pagination, Filters) {
var $t = $translate.instant;
$scope.minInstanceForPagination = Settings.minRowsToShowPagination;
$scope.clusterName = $routeParams.clusterName;
$scope.isLoading = false;
-
$scope.constants = {
groups: $t('common.clusters').toLowerCase()
};
-
- $scope.groupsPerPage = 10;
- $scope.currentPage = 1;
- $scope.totalGroups = 1;
- $scope.currentNameFilter = '';
- $scope.maxVisiblePages=20;
$scope.tableInfo = {
+ filtered: 0,
total: 0,
showed: 0
};
+ $scope.pagination = Pagination.create();
+ $scope.filters = [
+ {
+ key: 'clusterName',
+ label: $t('views.clusterName'),
+ options: []
+ },
+ {
+ key: 'service',
+ label: $t('common.services'),
+ customValueConverter: function (item) {
+ return item.ClusterInfo.services;
+ },
+ isMultiple: true,
+ options: []
+ }
+ ];
- $scope.isNotEmptyFilter = true;
-
- $scope.pageChanged = function() {
- loadRemoteClusters();
- };
- $scope.groupsPerPageChanges = function() {
- loadRemoteClusters();
+ $scope.toggleSearchBox = function () {
+ $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
};
- $scope.resetPagination = function() {
- $scope.currentPage = 1;
- loadRemoteClusters();
+ $scope.filterClusters = function (appliedFilters) {
+ $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.remoteClusters, $scope.filters);
+ $scope.pagination.resetPagination($scope.remoteClusters, $scope.tableInfo);
};
- $scope.typeFilterOptions = [
- $t('common.any')
- ];
-
- $scope.currentTypeFilter = $scope.typeFilterOptions[0];
-
- $scope.clearFilters = function () {
- $scope.currentNameFilter = '';
- $scope.currentTypeFilter = $scope.typeFilterOptions[0];
- $scope.resetPagination();
+ $scope.pageChanged = function () {
+ $scope.pagination.pageChanged($scope.remoteClusters, $scope.tableInfo);
};
- function loadRemoteClusters(){
- $scope.isLoading = true;
- RemoteCluster.all({
- currentPage: $scope.currentPage,
- groupsPerPage: $scope.groupsPerPage,
- searchString: $scope.currentNameFilter,
- service: $scope.currentTypeFilter
- }).then(function(remoteclusters) {
- $scope.isLoading = false;
-
- $scope.totalGroups = remoteclusters.itemTotal;
- $scope.tableInfo.total = remoteclusters.itemTotal;
- $scope.tableInfo.showed = remoteclusters.items.length;
-
- $scope.remoteClusters = remoteclusters.items;
-
- remoteclusters.items.map(function(clusteritem){
- clusteritem.ClusterInfo.services.map(function(service){
- var serviceIndex = $scope.typeFilterOptions.indexOf(service);
- if(serviceIndex == -1){
- $scope.typeFilterOptions.push(service);
- }
- })
- })
+ $scope.resetPagination = function () {
+ $scope.pagination.resetPagination($scope.remoteClusters, $scope.tableInfo);
+ };
- })
- .catch(function(data) {
+ function loadRemoteClusters() {
+ $scope.isLoading = true;
+ RemoteCluster.all().then(function (remoteclusters) {
+ $scope.isLoading = false;
+ $scope.remoteClusters = remoteclusters.items.map(function (item) {
+ item.clusterName = item.ClusterInfo.name;
+ return item;
+ });
+ $scope.tableInfo.total = $scope.remoteClusters.length;
+ $scope.filterClusters();
+ Filters.initFilterOptions($scope.filters, $scope.remoteClusters);
+ })
+ .catch(function (data) {
console.error($t('remoteClusters.alerts.fetchError'), data);
});
- };
+ }
loadRemoteClusters();
- $scope.$watch(
- function (scope) {
- return Boolean(scope.currentNameFilter || (scope.currentTypeFilter));
- },
- function (newValue, oldValue, scope) {
- scope.isNotEmptyFilter = newValue;
- }
- );
-
}]);
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
index a4b121c..3edaf07 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js
@@ -136,7 +136,7 @@ angular.module('ambariAdminConsole')
};
$scope.isDeletable = function() {
- return !($scope.repoStatus == 'current' || $scope.repoStatus == 'installed');
+ return !($scope.repoStatus === 'CURRENT' || $scope.repoStatus === 'INSTALLED');
};
$scope.disableUnusedOS = function() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
index ae00978..ea7a480 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js
@@ -18,7 +18,9 @@
'use strict';
angular.module('ambariAdminConsole')
- .controller('StackVersionsListCtrl', ['$scope', 'Cluster', 'Stack', '$routeParams', '$translate', 'Settings', function ($scope, Cluster, Stack, $routeParams, $translate, Settings) {
+ .controller('StackVersionsListCtrl',
+ ['$scope', 'Cluster', 'Stack', '$routeParams', '$translate', 'Settings', 'Pagination', '$q', 'Filters',
+ function ($scope, Cluster, Stack, $routeParams, $translate, Settings, Pagination, $q, Filters) {
var $t = $translate.instant;
$scope.getConstant = function (key) {
return $t(key).toLowerCase();
@@ -26,86 +28,86 @@ angular.module('ambariAdminConsole')
$scope.minInstanceForPagination = Settings.minRowsToShowPagination;
$scope.isLoading = false;
$scope.clusterName = $routeParams.clusterName;
- $scope.filter = {
- name: '',
- version: '',
- type: '',
- cluster: {
- options: [],
- current: null
- },
- stack: {
- options: [],
- current: null
- }
- };
- $scope.isNotEmptyFilter = true;
-
- $scope.pagination = {
- totalRepos: 10,
- maxVisiblePages: 20,
- itemsPerPage: 10,
- currentPage: 1
- };
-
$scope.tableInfo = {
total: 0,
showed: 0,
filtered: 0
};
-
$scope.repos = [];
$scope.dropDownClusters = [];
$scope.selectedCluster = $scope.dropDownClusters[0];
+ $scope.filters = [
+ {
+ key: 'stack',
+ label: $t('common.stack'),
+ customValueConverter: function(item) {
+ return item.stack_name + '-' + item.stack_version;
+ },
+ options: []
+ },
+ {
+ key: 'display_name',
+ label: $t('common.name'),
+ options: []
+ },
+ {
+ key: 'type',
+ label: $t('common.type'),
+ options: []
+ },
+ {
+ key: 'repository_version',
+ label: $t('common.version'),
+ options: []
+ },
+ {
+ key: 'cluster',
+ label: $t('common.cluster'),
+ options: []
+ }
+ ];
+ $scope.pagination = Pagination.create();
- $scope.resetPagination = function () {
- $scope.pagination.currentPage = 1;
- $scope.loadAllData();
+ $scope.resetPagination = function() {
+ $scope.pagination.resetPagination($scope.repos, $scope.tableInfo);
};
- $scope.pageChanged = function () {
- $scope.loadAllData();
+ $scope.pageChanged = function() {
+ $scope.pagination.pageChanged($scope.repos, $scope.tableInfo);
};
- $scope.goToCluster = function() {
- window.location.replace(Settings.siteRoot + '#/main/admin/stack/versions');
+ $scope.filterRepos = function (appliedFilters) {
+ $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.repos, $scope.filters);
+ $scope.pagination.resetPagination($scope.repos, $scope.tableInfo);
+ };
+
+ $scope.toggleSearchBox = function() {
+ $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
};
- $scope.clearFilters = function () {
- $scope.filter.name = '';
- $scope.filter.version = '';
- $scope.filter.cluster.current = $scope.filter.cluster.options[0];
- $scope.filter.stack.current = $scope.filter.stack.options[0];
- $scope.resetPagination();
+ $scope.goToCluster = function() {
+ window.location.replace(Settings.siteRoot + '#/main/admin/stack/versions');
};
$scope.fetchRepoClusterStatus = function (allRepos) {
+ var calls = [];
if (allRepos && allRepos.length) {
- var clusterName = ($scope.clusters && $scope.clusters.length > 0) ? $scope.clusters[0].Clusters.cluster_name : null, // only support one cluster at the moment
- repos = [],
- processedRepos = 0;
+ // only support one cluster at the moment
+ var clusterName = $scope.cluster && $scope.cluster.Clusters.cluster_name;
if (clusterName) {
- angular.forEach(allRepos, function (repo) {
- Cluster.getRepoVersionStatus(clusterName, repo.id).then(function (response) {
- repo.cluster = (response.status == 'current' || response.status == 'installed') ? clusterName : '';
- if (!$scope.filter.cluster.current.value || repo.cluster) {
+ $scope.repos = allRepos;
+ $scope.tableInfo.total = allRepos.length;
+ angular.forEach($scope.repos, function (repo) {
+ calls.push(Cluster.getRepoVersionStatus(clusterName, repo.id).then(function (response) {
+ repo.cluster = (response.status === 'CURRENT' || response.status === 'INSTALLED') ? clusterName : '';
+ if (repo.cluster) {
repo.status = response.status;
repo.totalHosts = response.totalHosts;
repo.currentHosts = response.currentHosts;
repo.installedHosts = response.installedHosts;
repo.stackVersionId = response.stackVersionId;
- repos.push(repo);
}
- processedRepos++;
- if (processedRepos === allRepos.length) {
- var from = ($scope.pagination.currentPage - 1) * $scope.pagination.itemsPerPage;
- var to = (repos.length - from > $scope.pagination.itemsPerPage) ? from + $scope.pagination.itemsPerPage : repos.length;
- $scope.repos = repos.slice(from, to);
- $scope.tableInfo.total = repos.length;
- $scope.pagination.totalRepos = repos.length;
- $scope.tableInfo.showed = to - from;
- }
- });
+ }));
});
}
} else {
@@ -114,101 +116,52 @@ angular.module('ambariAdminConsole')
$scope.pagination.totalRepos = 0;
$scope.tableInfo.showed = 0;
}
+ $scope.tableInfo.total = $scope.repos.length;
+ return $q.all(calls);
};
$scope.fetchRepos = function () {
- return Stack.allRepos($scope.filter).then(function (repos) {
+ return Stack.allRepos().then(function (repos) {
$scope.isLoading = false;
return repos.items;
});
};
- $scope.fillClusters = function (clusters) {
- $scope.dropDownClusters = [].concat(clusters);
- var options = [{label: $t('common.all'), value: ''}];
- angular.forEach(clusters, function (cluster) {
- options.push({
- label: cluster.Clusters.cluster_name,
- value: cluster.Clusters.cluster_name
- });
- });
- $scope.filter.cluster.options = options;
- if (!$scope.filter.cluster.current) {
- $scope.filter.cluster.current = options[0];
- }
- };
-
$scope.fetchClusters = function () {
return Cluster.getAllClusters().then(function (clusters) {
if (clusters && clusters.length > 0) {
- $scope.clusters = clusters;
- $scope.fillClusters(clusters);
- }
- });
- };
-
- $scope.fetchStacks = function () {
- return Stack.allStackVersions().then(function (clusters) {
- if (clusters && clusters.length > 0) {
- $scope.stacks = clusters;
- $scope.fillStacks(clusters);
- }
- });
- };
-
- $scope.fillStacks = function() {
- var options = [{label: $t('common.all'), value: ''}];
- angular.forEach($scope.stacks, function (stack) {
- if (stack.active) {
- options.push({
- label: stack.displayName,
- value: stack.displayName
- });
+ $scope.dropDownClusters = clusters;
}
});
- $scope.filter.stack.options = options;
- if (!$scope.filter.stack.current) {
- $scope.filter.stack.current = options[0];
- }
};
$scope.loadAllData = function () {
$scope.isLoading = true;
- $scope.fetchStacks()
- .then(function () {
- return $scope.fetchClusters();
- })
- .then(function () {
- return $scope.fetchRepos();
- })
+ $scope.fetchRepos()
.then(function (repos) {
- $scope.fetchRepoClusterStatus(repos);
+ $scope.fetchClusters();
+ $scope.fetchRepoClusterStatus(repos).then(function() {
+ Filters.initFilterOptions($scope.filters, $scope.repos);
+ });
+ $scope.filterRepos();
});
};
$scope.loadAllData();
- $scope.$watch('filter', function (filter) {
- $scope.isNotEmptyFilter = Boolean(filter.name
- || filter.version
- || filter.type
- || (filter.cluster.current && filter.cluster.current.value)
- || (filter.stack.current && filter.stack.current.value));
- }, true);
-
$scope.toggleVisibility = function (repo) {
repo.isProccessing = true;
var payload = {
- RepositoryVersions:{
+ RepositoryVersions: {
hidden: repo.hidden
}
- }
- Stack.updateRepo(repo.stack_name, repo.stack_version, repo.id, payload).then( null, function () {
+ };
+ Stack.updateRepo(repo.stack_name, repo.stack_version, repo.id, payload).then(null, function () {
repo.hidden = !repo.hidden;
- }).finally( function () {
+ }).finally(function () {
delete repo.isProccessing;
});
- }
+ };
$scope.isHideCheckBoxEnabled = function ( repo ) {
return !repo.isProccessing && ( !repo.cluster || repo.isPatch && ( repo.status === 'installed' || repo.status === 'install_failed') );
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/GroupsListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/GroupsListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/GroupsListCtrl.js
index 61b5282..f2a6f67 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/GroupsListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/GroupsListCtrl.js
@@ -19,8 +19,8 @@
angular.module('ambariAdminConsole')
.controller('GroupsListCtrl',
-['$scope', 'Group', '$modal', 'ConfirmationModal', '$rootScope', '$translate', 'Settings', 'Cluster', 'View', 'Alert',
-function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Settings, Cluster, View, Alert) {
+['$scope', 'Group', '$modal', 'ConfirmationModal', '$rootScope', '$translate', 'Settings', 'Cluster', 'View', 'Alert', 'Pagination', 'Filters',
+function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Settings, Cluster, View, Alert, Pagination, Filters) {
var $t = $translate.instant;
$scope.constants = {
groups: $t('common.groups').toLowerCase()
@@ -28,77 +28,57 @@ function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Setti
$scope.minRowsToShowPagination = Settings.minRowsToShowPagination;
$scope.isLoading = false;
$scope.groups = [];
-
- $scope.groupsPerPage = 10;
- $scope.currentPage = 1;
- $scope.totalGroups = 0;
- $scope.filter = {
- name: '',
- type: null
- };
- $scope.maxVisiblePages=20;
$scope.tableInfo = {
+ filtered: 0,
total: 0,
showed: 0
};
- $scope.isNotEmptyFilter = true;
+ $scope.pagination = Pagination.create();
+ $scope.filters = [
+ {
+ key: 'group_name',
+ label: $t('groups.name'),
+ options: []
+ },
+ {
+ key: 'groupTypeName',
+ label: $t('common.type'),
+ options: []
+ }
+ ];
+
+ $scope.resetPagination = function() {
+ $scope.pagination.resetPagination($scope.groups, $scope.tableInfo);
+ };
$scope.pageChanged = function() {
- loadGroups();
+ $scope.pagination.pageChanged($scope.groups, $scope.tableInfo);
};
- $scope.groupsPerPageChanges = function() {
- loadGroups();
+
+ $scope.filterGroups = function(appliedFilters) {
+ $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.groups, $scope.filters);
+ $scope.pagination.resetPagination($scope.groups, $scope.tableInfo);
};
- $scope.resetPagination = function() {
- $scope.currentPage = 1;
- loadGroups();
+ $scope.toggleSearchBox = function() {
+ $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
};
- function loadGroups(){
+ $scope.loadGroups = function() {
$scope.isLoading = true;
- Group.all({
- currentPage: $scope.currentPage,
- groupsPerPage: $scope.groupsPerPage,
- searchString: $scope.filter.name,
- group_type: $scope.filter.type.value
- }).then(function(groups) {
+ Group.all().then(function(groups) {
$scope.isLoading = false;
- $scope.totalGroups = groups.itemTotal;
$scope.groups = groups.map(Group.makeGroup);
- $scope.tableInfo.total = groups.itemTotal;
- $scope.tableInfo.showed = groups.length;
+ $scope.tableInfo.total = $scope.groups.length;
+ Filters.initFilterOptions($scope.filters, $scope.groups);
+ $scope.filterGroups();
})
.catch(function(data) {
Alert.error($t('groups.alerts.getGroupsListError'), data.data.message);
});
- }
-
- $scope.typeFilterOptions = [{ label: $t('common.all'), value: '*'}]
- .concat(Object.keys(Group.getTypes()).map(function(key) {
- return {
- label: $t(Group.getTypes()[key].LABEL_KEY),
- value: Group.getTypes()[key].VALUE
- };
- }));
- $scope.filter.type = $scope.typeFilterOptions[0];
-
- $scope.clearFilters = function () {
- $scope.filter.name = '';
- $scope.filter.type = $scope.typeFilterOptions[0];
- $scope.resetPagination();
};
-
- loadGroups();
- $scope.$watch(
- function (scope) {
- return Boolean(scope.filter.name || (scope.filter.type && scope.filter.type.value !== '*'));
- },
- function (newValue, oldValue, scope) {
- scope.isNotEmptyFilter = newValue;
- }
- );
+ $scope.loadGroups();
$rootScope.$watch(function(scope) {
return scope.LDAPSynced;
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/UsersListCtrl.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/UsersListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/UsersListCtrl.js
index 00bf9c3..7982f0b 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/UsersListCtrl.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/UsersListCtrl.js
@@ -19,95 +19,105 @@
angular.module('ambariAdminConsole')
.controller('UsersListCtrl',
-['$scope', 'User', '$modal', '$rootScope', 'UserConstants', '$translate', 'Cluster', 'View', 'ConfirmationModal', 'Settings',
-function($scope, User, $modal, $rootScope, UserConstants, $translate, Cluster, View, ConfirmationModal, Settings) {
+['$scope', 'User', '$modal', '$rootScope', 'UserConstants', '$translate', 'Cluster', 'View', 'ConfirmationModal', 'Settings', 'Pagination', 'Filters',
+function($scope, User, $modal, $rootScope, UserConstants, $translate, Cluster, View, ConfirmationModal, Settings, Pagination, Filters) {
var $t = $translate.instant;
$scope.constants = {
admin: $t('users.ambariAdmin'),
users: $t('common.users').toLowerCase()
};
+ $scope.users = [];
$scope.minRowsToShowPagination = Settings.minRowsToShowPagination;
$scope.isLoading = false;
- $scope.users = [];
- $scope.usersPerPage = 10;
- $scope.currentPage = 1;
- $scope.totalUsers = 0;
- $scope.filters = {
- name: '',
- status: null,
- type: null
- };
- $scope.maxVisiblePages = 20;
+ $scope.pagination = Pagination.create();
$scope.tableInfo = {
+ filtered: 0,
total: 0,
showed: 0
};
- $scope.isNotEmptyFilter = true;
+ $scope.filters = [
+ {
+ key: 'user_name',
+ label: $t('users.username'),
+ customValueConverter: function(item) {
+ return item.Users.user_name;
+ },
+ options: []
+ },
+ {
+ key: 'role',
+ label: $t('clusters.role'),
+ customValueConverter: function(item) {
+ return item.Users.roles[0] ? item.Users.roles[0].permission_label : '';
+ },
+ options: []
+ },
+ {
+ key: 'status',
+ label: $t('users.status'),
+ isStatic: true,
+ customValueConverter: function(item) {
+ return item.Users.active ? $t('users.active') : $t('users.inactive');
+ },
+ options: [
+ {
+ key: $t('users.active'),
+ label: $t('users.active')
+ },
+ {
+ key: $t('users.inactive'),
+ label: $t('users.inactive')
+ }
+ ]
+ },
+ {
+ key: 'type',
+ label: $t('common.type'),
+ customValueConverter: function(item) {
+ return item.Users.userTypeName;
+ },
+ options: []
+ },
+ {
+ key: 'group',
+ label: $t('common.group'),
+ isMultiple: true,
+ customValueConverter: function(item) {
+ return item.Users.groups;
+ },
+ options: []
+ }
+ ];
function loadUsers() {
$scope.isLoading = true;
- User.list({
- currentPage: $scope.currentPage,
- usersPerPage: $scope.usersPerPage,
- searchString: $scope.filters.name,
- user_type: $scope.filters.type.value,
- active: $scope.filters.status.value
- }).then(function (data) {
- $scope.totalUsers = data.data.itemTotal;
+ User.list().then(function (data) {
$scope.users = data.data.items.map(User.makeUser);
- $scope.tableInfo.showed = data.data.items.length;
- $scope.tableInfo.total = data.data.itemTotal;
+ $scope.tableInfo.total = $scope.users.length;
+ $scope.filterUsers();
+ Filters.initFilterOptions($scope.filters, $scope.users);
}).finally(function () {
$scope.isLoading = false;
});
}
- $scope.pageChanged = function () {
- loadUsers();
+ $scope.toggleSearchBox = function() {
+ $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
};
- $scope.usersPerPageChanges = function () {
- $scope.resetPagination();
+
+ $scope.pageChanged = function () {
+ $scope.pagination.pageChanged($scope.users, $scope.tableInfo);
};
$scope.resetPagination = function () {
- $scope.currentPage = 1;
- loadUsers();
+ $scope.pagination.resetPagination($scope.users, $scope.tableInfo);
};
- $scope.activeFilterOptions = [
- {label: $t('common.all'), value: '*'},
- {label: $t('users.active'), value: true},
- {label: $t('users.inactive'), value: false}
- ];
- $scope.filters.status = $scope.activeFilterOptions[0];
-
- $scope.typeFilterOptions = [{label: $t('common.all'), value: '*'}]
- .concat(Object.keys(UserConstants.TYPES).map(function (key) {
- return {
- label: $t(UserConstants.TYPES[key].LABEL_KEY),
- value: UserConstants.TYPES[key].VALUE
- };
- }));
-
- $scope.filters.type = $scope.typeFilterOptions[0];
-
- $scope.clearFilters = function () {
- $scope.filters.name = '';
- $scope.filters.type = $scope.typeFilterOptions[0];
- $scope.filters.status = $scope.activeFilterOptions[0];
- $scope.resetPagination();
+ $scope.filterUsers = function(appliedFilters) {
+ $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.users, $scope.filters);
+ $scope.pagination.resetPagination($scope.users, $scope.tableInfo);
};
- $scope.$watch(
- function (scope) {
- return Boolean(scope.filters.name || (scope.filters.status && scope.filters.status.value !== '*')
- || (scope.filters.type && scope.filters.type.value !== '*'));
- },
- function (newValue, oldValue, scope) {
- scope.isNotEmptyFilter = newValue;
- }
- );
-
$rootScope.$watch(function (scope) {
return scope.LDAPSynced;
}, function (LDAPSynced) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
index af25167..fc58eae 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js
@@ -169,10 +169,8 @@ angular.module('ambariAdminConsole')
return !(option.key === '' || option.key === undefined || appliedOptions[option.key])
&& (!filter.searchOptionInput || option.label.toLowerCase().indexOf(filter.searchOptionInput.toLowerCase()) !== -1);
});
+ resetActive(filter.filteredOptions);
filter.showAutoComplete = filter.filteredOptions.length > 0;
- if (filter.filteredOptions.length > 0) {
- $scope.makeActive(filter.filteredOptions[0], filter.filteredOptions);
- }
};
$scope.extractFilters = function(filters) {
@@ -247,7 +245,13 @@ angular.module('ambariAdminConsole')
return i;
}
}
- return 0;
+ return -1;
+ }
+
+ function resetActive(array) {
+ array.forEach(function(item) {
+ item.active = false;
+ });
}
function focusInput(filter) {
@@ -282,6 +286,10 @@ angular.module('ambariAdminConsole')
leftArrowKeyHandler();
$scope.$apply();
}
+ if (event.which === 27) { // "Escape" key
+ $scope.showAutoComplete = false;
+ $scope.$apply();
+ }
});
}
@@ -341,7 +349,14 @@ angular.module('ambariAdminConsole')
if (activeAppliedFilters.length > 0) {
var filteredOptions = activeAppliedFilters[0].filteredOptions;
activeIndex = findActiveByProperty(filteredOptions);
- nextIndex = (activeIndex < filteredOptions.length - 1) ? activeIndex + 1 : 0;
+ if (activeIndex < filteredOptions.length - 1) {
+ nextIndex = activeIndex + 1;
+ } else {
+ //switch to input of option
+ nextIndex = null;
+ resetActive(filteredOptions);
+ focusInput(activeAppliedFilters[0]);
+ }
}
if (nextIndex !== null) {
$scope.makeActive(filteredOptions[nextIndex], filteredOptions);
@@ -374,7 +389,16 @@ angular.module('ambariAdminConsole')
if (activeAppliedFilters.length > 0) {
var filteredOptions = activeAppliedFilters[0].filteredOptions;
activeIndex = findActiveByProperty(filteredOptions);
- nextIndex = (activeIndex > 0) ? activeIndex - 1 : filteredOptions.length - 1;
+ if (activeIndex > 0) {
+ nextIndex = activeIndex - 1;
+ } else if (activeIndex === 0) {
+ //switch to input of option
+ nextIndex = null;
+ resetActive(filteredOptions);
+ focusInput(activeAppliedFilters[0]);
+ } else {
+ nextIndex = filteredOptions.length - 1;
+ }
}
if (nextIndex !== null) {
$scope.makeActive(filteredOptions[nextIndex], filteredOptions);
@@ -401,7 +425,8 @@ angular.module('ambariAdminConsole')
if (activeOptions.length > 0) {
$scope.selectOption(null, activeOptions[0], activeAppliedFilters[0]);
}
- } else {
+ }
+ if (activeAppliedFilters.length === 0 || activeOptions.length === 0) {
$scope.appliedFilters.filter(function(item) {
return !item.currentOption;
}).forEach(function(item) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Cluster.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Cluster.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Cluster.js
index 30ef91a..1a99a65 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Cluster.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Cluster.js
@@ -292,8 +292,7 @@ angular.module('ambariAdminConsole')
angular.forEach(hostStatus, function(status) {
totalHosts += status.length;
});
- response.status = currentHosts > 0? 'current' :
- installedHosts > 0? 'installed' : '';
+ response.status = data[0].ClusterStackVersions.state;
response.currentHosts = currentHosts;
response.installedHosts = installedHosts;
response.totalHosts = totalHosts;
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Filters.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Filters.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Filters.js
new file mode 100644
index 0000000..7c6815f
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Filters.js
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+angular.module('ambariAdminConsole')
+.factory('Filters', function() {
+
+ function initFilterOptions(filters, items) {
+ filters.filter(function(filter) {
+ return !filter.isStatic;
+ }).forEach(function(filter) {
+ var preOptions = [];
+ if (filter.isMultiple) {
+ items.forEach(function(item) {
+ if (typeof filter.customValueConverter === 'function') {
+ preOptions = preOptions.concat(filter.customValueConverter(item));
+ } else {
+ preOptions = preOptions.concat(item[filter.key]);
+ }
+ });
+ } else {
+ preOptions = items.map(function(item) {
+ if (typeof filter.customValueConverter === 'function') {
+ return filter.customValueConverter(item);
+ }
+ return item[filter.key];
+ });
+ }
+ filter.options = $.unique(preOptions).filter(function(item) {
+ return item !== undefined && item !== null;
+ }).map(function(item) {
+ return {
+ key: item,
+ label: item
+ }
+ });
+ });
+ }
+
+ function filterItems(appliedFilters, items, filterDefinitions) {
+ var filteredCount = 0;
+ angular.forEach(items, function(item) {
+ item.isFiltered = !(appliedFilters && appliedFilters.length > 0 && appliedFilters.some(function(filter) {
+ var customValueFilter = filterDefinitions.filter(function(filterDefinition) {
+ return filterDefinition.key === filter.key && typeof filterDefinition.customValueConverter === 'function';
+ })[0];
+ if (customValueFilter) {
+ return filter.values.every(function(value) {
+ var itemValue = customValueFilter.customValueConverter(item);
+ return String(Array.isArray(itemValue) ? itemValue.join() : itemValue).indexOf(value) === -1;
+ });
+ }
+ return filter.values.every(function(value) {
+ var itemValue = item[filter.key];
+ return String(Array.isArray(itemValue) ? itemValue.join() : itemValue).indexOf(value) === -1;
+
+ });
+ }));
+
+ filteredCount += ~~item.isFiltered;
+ });
+ return filteredCount;
+ }
+
+ return {
+ initFilterOptions: initFilterOptions,
+ filterItems: filterItems
+ };
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Group.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Group.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Group.js
index 0509e11..dc6d351 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Group.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Group.js
@@ -100,18 +100,11 @@ angular.module('ambariAdminConsole')
return $http.post(Settings.baseUrl + '/groups/' + groupName + '/members/'+memberName);
};
- Group.all = function(params) {
+ Group.all = function() {
var deferred = $q.defer();
- $http.get(Settings.baseUrl + '/groups?'
- + 'Groups/group_name.matches(.*'+params.searchString+'.*)'
- + '&fields=*'
- + '&from='+ (params.currentPage-1)*params.groupsPerPage
- + '&page_size=' + params.groupsPerPage
- + (params.group_type === '*' ? '' : '&Groups/group_type=' + params.group_type)
- )
+ $http.get(Settings.baseUrl + '/groups?fields=*')
.success(function(data) {
- data.items.itemTotal = data.itemTotal;
deferred.resolve(data.items);
})
.error(function(data) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Pagination.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Pagination.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Pagination.js
new file mode 100644
index 0000000..ac0a8bc
--- /dev/null
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Pagination.js
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+angular.module('ambariAdminConsole')
+.factory('Pagination', function() {
+
+ function showItemsOnPage(items, tableInfo) {
+ var startIndex = (this.currentPage - 1) * this.itemsPerPage + 1;
+ var endIndex = this.currentPage * this.itemsPerPage;
+ var showedCount = 0;
+ var filteredCount = 0;
+
+ angular.forEach(items, function (item) {
+ item.isShowed = false;
+ if (item.isFiltered) {
+ filteredCount++;
+ if (filteredCount >= startIndex && filteredCount <= endIndex) {
+ item.isShowed = true;
+ showedCount++;
+ }
+ }
+ });
+ tableInfo.showed = showedCount;
+ }
+
+ return {
+ create: function(options) {
+ options = options || {};
+ return {
+ itemsPerPage: options.itemsPerPage || 10,
+ currentPage: options.currentPage || 1,
+ maxVisiblePages: options.maxVisiblePages || 10,
+ pageChanged: function(items, tableInfo) {
+ showItemsOnPage.call(this, items, tableInfo);
+ },
+ resetPagination: function(items, tableInfo) {
+ this.currentPage = 1;
+ showItemsOnPage.call(this, items, tableInfo);
+ }
+ }
+ }
+ };
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/RemoteCluster.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/RemoteCluster.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/RemoteCluster.js
index 49c6abc..86407fd 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/RemoteCluster.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/RemoteCluster.js
@@ -80,16 +80,10 @@ angular.module('ambariAdminConsole')
return deferred.promise;
}
- RemoteCluster.all = function(params) {
+ RemoteCluster.all = function() {
var deferred = $q.defer();
- $http.get(Settings.baseUrl + "/remoteclusters?"
- + 'ClusterInfo/name.matches(.*'+params.searchString+'.*)'
- + '&fields=*'
- + '&from='+ (params.currentPage-1)*params.groupsPerPage
- + '&page_size=' + params.groupsPerPage
- + (params.service === 'Any' ? '' : '&ClusterInfo/services.matches(.*'+params.service+'.*)')
- )
+ $http.get(Settings.baseUrl + "/remoteclusters")
.success(function(response) {
deferred.resolve(response);
})
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
index f52e687..4f111fe 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js
@@ -189,28 +189,8 @@ angular.module('ambariAdminConsole')
}
},
- allRepos: function (filter, pagination) {
- var versionFilter = filter.version;
- var nameFilter = filter.name;
- var typeFilter = filter.type;
- var stackFilter = filter.stack && filter.stack.current && filter.stack.current.value;
+ allRepos: function () {
var url = '/stacks?fields=versions/repository_versions/RepositoryVersions';
- if (versionFilter) {
- url += '&versions/repository_versions/RepositoryVersions/repository_version.matches(.*' + versionFilter + '.*)';
- }
- if (nameFilter) {
- url += '&versions/repository_versions/RepositoryVersions/display_name.matches(.*' + nameFilter + '.*)';
- }
- if (typeFilter){
- url += '&versions/repository_versions/RepositoryVersions/type.matches(.*' + typeFilter.toUpperCase() + '.*)';
- }
- if (stackFilter) {
- var stack = filter.stack.current.value.split('-'),
- stackNameFilter = stack[0],
- stackVersionFilter = stack[1];
- url += '&versions/repository_versions/RepositoryVersions/stack_name=' + stackNameFilter;
- url += '&versions/repository_versions/RepositoryVersions/stack_version=' + stackVersionFilter;
- }
var deferred = $q.defer();
$http.get(Settings.baseUrl + url, {mock: 'version/versions.json'})
.success(function (data) {
@@ -230,16 +210,8 @@ angular.module('ambariAdminConsole')
});
// prepare response data with client side pagination
var response = {};
+ response.items = repos;
response.itemTotal = repos.length;
- if (pagination) {
- var from = (pagination.currentPage - 1) * pagination.itemsPerPage;
- var to = (repos.length - from > pagination.itemsPerPage)? from + pagination.itemsPerPage : repos.length;
- response.items = repos.slice(from, to);
- response.showed = to - from;
- } else {
- response.items = repos;
- response.showed = repos.length;
- }
deferred.resolve(response);
})
.error(function (data) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/User.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/User.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/User.js
index 7932d9b..5ed692a 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/User.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/User.js
@@ -33,17 +33,8 @@ angular.module('ambariAdminConsole')
var $t = $translate.instant;
return {
- list: function(params) {
- return $http.get(
- Settings.baseUrl + '/users/?'
- + 'Users/user_name.matches(.*'+params.searchString+'.*)'
- + '&fields=privileges/PrivilegeInfo/*,Users'
- + '&from=' + (params.currentPage-1)*params.usersPerPage
- + '&page_size=' + params.usersPerPage
- + (params.user_type === '*' ? '' : '&Users/user_type=' + params.user_type)
- + (params.active === '*' ? '' : '&Users/active=' + params.active)
- + (params.admin ? '&Users/admin=true' : '')
- );
+ list: function() {
+ return $http.get(Settings.baseUrl + '/users?fields=Users/*,privileges/*');
},
listByName: function(name) {
return $http.get(
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/styles/user-management.css
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/styles/user-management.css b/ambari-admin/src/main/resources/ui/admin-web/app/styles/user-management.css
index 3c9756e..bca53ca 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/styles/user-management.css
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/styles/user-management.css
@@ -20,6 +20,10 @@
vertical-align: baseline;
}
+#user-management .search-box-row {
+ margin-top: -1px;
+}
+
#user-management .nav.nav-tabs {
margin-bottom: 0;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
index 9e9cb55..48f445b 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html
@@ -115,13 +115,12 @@
<div class="col-sm-12 table-bar" ng-show="instances.length >= minInstanceForPagination">
<div class="pull-left filtered-info">
<span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.filtered, term: urs.urls} }}</span>
- <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
</div>
<div class="pull-right left-margin">
- <pagination class="paginator" total-items="tableInfo.filtered" max-size="maxVisiblePages" items-per-page="instancesPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+ <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="pagination.itemsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
</div>
<div class="pull-right">
- <select class="form-control" ng-model="instancesPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+ <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
index 7a8e6f4..5f8a0be 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html
@@ -23,30 +23,34 @@
{{'views.registerRemoteCluster' | translate}}
</a>
</div>
+ <div class="search-box-button pull-right">
+ <button class="btn btn-default" ng-click="toggleSearchBox()">
+ <i class="fa fa-filter" aria-hidden="true"></i>
+ </button>
+ <div class="popup-arrow-up hide"></div>
+ </div>
+ </div>
+
+ <div class="search-box-row hide">
+ <combo-search suggestions="filters" filter-change="filterClusters" placeholder="Search"></combo-search>
</div>
+
<table class="table table-striped table-hover">
<thead>
<tr>
<th class="col-sm-3">
<div class="search-container">
- <label for="">{{'views.clusterName' | translate}}</label>
- <input type="text" class="form-control namefilter" placeholder="{{'common.any' | translate}}" ng-model="currentNameFilter" ng-change="resetPagination()">
- <button type="button" class="close" ng-show="currentNameFilter" ng-click="currentNameFilter=''; resetPagination()"><span aria-hidden="true">×</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
+ <label>{{'views.clusterName' | translate}}</label>
</div>
</th>
<th class="col-sm-9">
- <label for="">{{'common.services' | translate}}</label>
- <select class="form-control typefilter"
- ng-model="currentTypeFilter"
- ng-options="item for item in typeFilterOptions"
- ng-change="resetPagination();">
- </select>
+ <label>{{'common.services' | translate}}</label>
</th>
</tr>
</thead>
<tbody>
- <tr ng-repeat="remoteCluster in remoteClusters" >
- <td class="col-sm-3"><a href="#/remoteClusters/{{remoteCluster.ClusterInfo.name}}/edit">{{ remoteCluster.ClusterInfo.name }}</a></td>
+ <tr ng-repeat="remoteCluster in remoteClusters | filter : { isShowed: true }">
+ <td class="col-sm-3"><a href="#/remoteClusters/{{remoteCluster.ClusterInfo.name}}/edit">{{ remoteCluster.clusterName }}</a></td>
<td class="col-sm-9">
<span ng-repeat="service in remoteCluster.ClusterInfo.services" ng-if="remoteCluster.ClusterInfo.services.length > 0">{{ service }}{{$last ? '' : ','}} </span>
<span ng-if="remoteCluster.ClusterInfo.services.length == 0">--</span>
@@ -58,19 +62,18 @@
<div ng-if="isLoading" class="spinner-container">
<i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
</div>
- <div class="alert empty-table-alert col-sm-12" ng-show="!remoteClusters.length && !isLoading">
+ <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
{{'common.alerts.noRemoteClusterDisplay' | translate}}
</div>
<div class="col-sm-12 table-bar" ng-show="tableInfo.total >= minInstanceForPagination">
<div class="pull-left filtered-info">
<span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.groups} }}</span>
- <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
</div>
<div class="pull-right left-margin">
- <pagination class="paginator" total-items="totalGroups" max-size="maxVisiblePages" items-per-page="groupsPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+ <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="groupsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
</div>
<div class="pull-right">
- <select class="form-control" ng-model="groupsPerPage" ng-change="groupsPerPageChanges()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+ <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
index 9d81543..697b3b2 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html
@@ -23,47 +23,42 @@
{{'versions.register.title' | translate}}
</a>
</div>
+ <div class="search-box-button pull-right">
+ <button class="btn btn-default" ng-click="toggleSearchBox()">
+ <i class="fa fa-filter" aria-hidden="true"></i>
+ </button>
+ <div class="popup-arrow-up hide"></div>
+ </div>
+ </div>
+
+ <div class="search-box-row hide">
+ <combo-search suggestions="filters" filter-change="filterRepos" placeholder="Search"></combo-search>
</div>
+
<table class="table table-striped table-hover">
<thead>
<tr>
<th class="col-small">
<label>{{'common.stack' | translate}}</label>
- <select class="form-control"
- ng-change="resetPagination()"
- ng-model="filter.stack.current"
- ng-options="item.label for item in filter.stack.options track by item.value"
- ></select>
</th>
<th class="col-medium text-search-container">
<label>{{'common.name' | translate}}</label>
- <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.name" placeholder="{{'common.any' | translate}}">
- <button type="button" class="close clearfilter" ng-show="filter.name" ng-click="filter.name=''; resetPagination()"><span aria-hidden="true">×</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
</th>
<th class="col-medium text-search-container">
<label>{{'common.type' | translate}}</label>
- <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.type" placeholder="{{'common.any' | translate}}">
- <button type="button" class="close clearfilter" ng-show="filter.type" ng-click="filter.type=''; resetPagination()"><span aria-hidden="true">×</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
</th>
<th class="col-medium text-search-container">
<label>{{'common.version' | translate}}</label>
- <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.version" placeholder="{{'common.any' | translate}}">
- <button type="button" class="close clearfilter" ng-show="filter.version" ng-click="filter.version=''; resetPagination()"><span aria-hidden="true">×</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
</th>
<th class="col-small">
<label>{{'common.cluster' | translate}}</label>
- <select class="form-control"
- ng-change="resetPagination()"
- ng-model="filter.cluster.current"
- ng-options="item.label for item in filter.cluster.options track by item.value"
- ></select>
</th>
<th class="col-small"></th>
<th class="col-small text-center vertical-top">{{'common.hidden' | translate}}</th>
</tr>
</thead>
<tbody>
- <tr ng-repeat="repo in repos">
+ <tr ng-repeat="repo in repos | filter : { isShowed: true }">
<td class="col-small">
<span>{{repo.stack_name}}-{{repo.stack_version}}</span>
</td>
@@ -120,17 +115,16 @@
<div ng-if="isLoading" class="spinner-container">
<i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
</div>
- <div class="alert empty-table-alert col-sm-12" ng-show="!repos.length && !isLoading">
+ <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
{{'common.alerts.nothingToDisplay' | translate:{term: getConstant("common.version")} }}
</div>
<div class="col-sm-12 table-bar" ng-show="tableInfo.total >= minInstanceForPagination">
<div class="pull-left filtered-info">
<span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: getConstant("common.versions")} }}</span>
- <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
</div>
<div class="pull-right left-margin">
<pagination class="paginator"
- total-items="pagination.totalRepos"
+ total-items="tableInfo.total"
max-size="pagination.maxVisiblePages"
items-per-page="pagination.itemsPerPage"
ng-model="pagination.currentPage"
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupsList.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupsList.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupsList.html
index af8303d..4218a65 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupsList.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupsList.html
@@ -16,11 +16,24 @@
* limitations under the License.
-->
<div class="groups-pane">
- <div class="clearfix panel">
- <button class="btn btn-default creategroup-btn pull-right" ng-click="createGroup()">
- {{'groups.createLocal' | translate}}
- </button>
+ <div class="clearfix">
+ <div class="pull-right">
+ <button class="btn btn-default creategroup-btn pull-right" ng-click="createGroup()">
+ {{'groups.createLocal' | translate}}
+ </button>
+ </div>
+ <div class="search-box-button pull-right">
+ <button class="btn btn-default" ng-click="toggleSearchBox()">
+ <i class="fa fa-filter" aria-hidden="true"></i>
+ </button>
+ <div class="popup-arrow-up hide"></div>
+ </div>
</div>
+
+ <div class="search-box-row hide">
+ <combo-search suggestions="filters" filter-change="filterGroups" placeholder="Search"></combo-search>
+ </div>
+
<table class="table table-striped table-hover col-sm-12">
<thead>
<tr>
@@ -37,26 +50,9 @@
<span>{{'common.actions' | translate}}</span>
</th>
</tr>
- <tr>
- <th class="col-sm-6">
- <div class="search-container">
- <input type="text" class="form-control namefilter" placeholder="{{'common.any' | translate}}" ng-model="filter.name" ng-change="resetPagination()">
- <button type="button" class="close" ng-show="filter.name" ng-click="filter.name=''; resetPagination()"><span aria-hidden="true">×</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
- </div>
- </th>
- <th class="col-sm-2">
- <select class="form-control typefilter"
- ng-model="filter.type"
- ng-options="item.label for item in typeFilterOptions"
- ng-change="resetPagination();">
- </select>
- </th>
- <th class="col-sm-2"></th>
- <th class="col-sm-2"></th>
- </tr>
</thead>
<tbody>
- <tr ng-repeat="group in groups">
+ <tr ng-repeat="group in groups | filter : { isShowed: true }">
<td class="col-sm-8">
<span>{{group.group_name}}</span>
</td>
@@ -76,19 +72,18 @@
<div ng-if="isLoading" class="spinner-container">
<i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
</div>
- <div class="alert empty-table-alert col-sm-12" ng-show="!groups.length && !isLoading">
+ <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
{{'common.alerts.nothingToDisplay' | translate:{term: constants.groups} }}
</div>
- <div class="col-sm-12 table-bar" ng-show="totalGroups > minRowsToShowPagination">
+ <div class="col-sm-12 table-bar" ng-show="tableInfo.total > minRowsToShowPagination">
<div class="pull-left filtered-info">
- <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.groups} }}</span>
- <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
+ <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.filtered, term: constants.groups} }}</span>
</div>
<div class="pull-right left-margin">
- <pagination class="paginator" total-items="totalGroups" max-size="maxVisiblePages" items-per-page="groupsPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+ <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="pagination.itemsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
</div>
<div class="pull-right">
- <select class="form-control" ng-model="groupsPerPage" ng-change="groupsPerPageChanges()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+ <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
</div>
</div>
</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/usersList.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/usersList.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/usersList.html
index 23e9ddb..2348dad 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/usersList.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/usersList.html
@@ -17,14 +17,27 @@
-->
<div class="users-pane">
- <div class="clearfix panel">
- <button class="btn btn-default createuser-btn pull-right" ng-click="createUser();">
- {{'users.create' | translate}}
- </button>
+ <div class="clearfix">
+ <div class="pull-right">
+ <button class="btn btn-default createuser-btn pull-right" ng-click="createUser();">
+ {{'users.create' | translate}}
+ </button>
+ </div>
+ <div class="search-box-button pull-right">
+ <button class="btn btn-default" ng-click="toggleSearchBox()">
+ <i class="fa fa-filter" aria-hidden="true"></i>
+ </button>
+ <div class="popup-arrow-up hide"></div>
+ </div>
</div>
+
+ <div class="search-box-row hide">
+ <combo-search suggestions="filters" filter-change="filterUsers" placeholder="Search"></combo-search>
+ </div>
+
<table class="table table-striped table-hover">
<thead>
- <tr class="fix-bottom">
+ <tr>
<th>
<span>{{'users.username' | translate}}</span>
</th>
@@ -44,36 +57,9 @@
<span>{{'common.actions' | translate}}</span>
</th>
</tr>
- <tr class="fix-top">
- <th>
- <div class="search-container">
- <input type="text" class="form-control namefilter" placeholder="{{'common.any' | translate}}" ng-model="filters.name" ng-change="resetPagination()">
- <button type="button" class="close clearfilter" ng-show="filters.name" ng-click="filters.name=''; resetPagination()">
- <span aria-hidden="true">×</span><span class="sr-only">{{'common.controls.close' | translate}}</span>
- </button>
- </div>
- </th>
- <th></th>
- <th>
- <select class="form-control statusfilter"
- ng-model="filters.status"
- ng-options="item.label for item in activeFilterOptions"
- ng-change="resetPagination()">
- </select>
- </th>
- <th>
- <select class="form-control typefilter"
- ng-model="filters.type"
- ng-options="item.label for item in typeFilterOptions"
- ng-change="resetPagination()">
- </select>
- </th>
- <th></th>
- <th></th>
- </tr>
</thead>
<tbody>
- <tr ng-repeat="user in users">
+ <tr ng-repeat="user in users | filter : { isShowed: true }">
<td>
<span>{{user.Users.user_name}}</span>
</td>
@@ -101,19 +87,18 @@
<div ng-if="isLoading" class="spinner-container">
<i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
</div>
- <div class="alert empty-table-alert col-sm-12" ng-show="!users.length && !isLoading">
+ <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
{{'common.alerts.nothingToDisplay' | translate:{term: constants.users} }}
</div>
- <div class="col-sm-12 table-bar" ng-show="totalUsers > minRowsToShowPagination">
+ <div class="col-sm-12 table-bar" ng-show="users.length > minRowsToShowPagination">
<div class="pull-left filtered-info">
<span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.users} }}</span>
- <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
</div>
<div class="pull-right left-margin">
- <pagination class="paginator" total-items="totalUsers" max-size="maxVisiblePages" items-per-page="usersPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+ <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="pagination.itemsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
</div>
<div class="pull-right">
- <select class="form-control" ng-model="usersPerPage" ng-change="usersPerPageChanges()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+ <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
index 362b94a..ca8205c 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js
@@ -24,6 +24,9 @@ describe('#Cluster', function () {
module('ambariAdminConsole');
inject(function($rootScope, $controller) {
scope = $rootScope.$new();
+ scope.pagination = {
+ resetPagination: angular.noop
+ };
ctrl = $controller('ViewsListCtrl', {$scope: scope});
});
scope.instances = [
@@ -44,68 +47,9 @@ describe('#Cluster', function () {
];
});
- describe('#initFilterOptions()', function () {
- beforeEach(function() {
- scope.initFilterOptions();
- });
-
- it('should fill short_url_name options', function() {
- expect(scope.filters[0].options).toEqual([
- {
- key: 'sun1',
- label: 'sun1'
- },
- {
- key: 'sun2',
- label: 'sun2'
- }
- ]);
- });
-
- it('should fill url options', function() {
- expect(scope.filters[1].options).toEqual([
- {
- key: '/main/view/vn1/su1',
- label: '/main/view/vn1/su1'
- },
- {
- key: '/main/view/vn2/su2',
- label: '/main/view/vn2/su2'
- }
- ]);
- });
-
- it('should fill view_name options', function() {
- expect(scope.filters[2].options).toEqual([
- {
- key: 'vn1',
- label: 'vn1'
- },
- {
- key: 'vn2',
- label: 'vn2'
- }
- ]);
- });
-
- it('should fill instance_name options', function() {
- expect(scope.filters[3].options).toEqual([
- {
- key: 'in1',
- label: 'in1'
- },
- {
- key: 'in2',
- label: 'in2'
- }
- ]);
- });
- });
-
-
describe('#filterInstances', function() {
beforeEach(function() {
- spyOn(scope, 'resetPagination');
+ spyOn(scope.pagination, 'resetPagination');
});
it('all should be filtered when filters not applied', function() {
@@ -117,7 +61,7 @@ describe('#Cluster', function () {
it('resetPagination should be called', function() {
scope.filterInstances();
- expect(scope.resetPagination).toHaveBeenCalled();
+ expect(scope.pagination.resetPagination).toHaveBeenCalled();
});
it('one view should be filtered', function() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/81c04545/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/GroupsListCtrl_test.js
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/GroupsListCtrl_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/GroupsListCtrl_test.js
index 8d04757..1e76aec 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/GroupsListCtrl_test.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/GroupsListCtrl_test.js
@@ -20,110 +20,20 @@ describe('#Cluster', function () {
describe('GroupsListCtrl', function() {
- var scope, ctrl, $t, $httpBackend;
+ var $scope, ctrl, $t, $httpBackend, $group;
beforeEach(module('ambariAdminConsole', function () {}));
- beforeEach(inject(function($rootScope, $controller, _$translate_, _$httpBackend_) {
- scope = $rootScope.$new();
+ beforeEach(inject(function($rootScope, $controller, _$translate_, _$httpBackend_, _Group_) {
+ $scope = $rootScope.$new();
$t = _$translate_.instant;
$httpBackend = _$httpBackend_;
+ $group = _Group_;
ctrl = $controller('GroupsListCtrl', {
- $scope: scope
+ $scope: $scope
});
}));
- describe('#clearFilters()', function () {
-
- it('should clear filters and reset pagination', function () {
- scope.currentPage = 2;
- scope.filter.name = 'a';
- scope.filter.type = {
- label: $t('common.local'),
- value: false
- };
- scope.clearFilters();
- expect(scope.filter.name).toEqual('');
- expect(scope.filter.type).toEqual({
- label: $t('common.all'),
- value: '*'
- });
- expect(scope.currentPage).toEqual(1);
- });
-
- });
-
- describe('#isNotEmptyFilter', function () {
-
- var cases = [
- {
- currentNameFilter: '',
- currentTypeFilter: null,
- isNotEmptyFilter: false,
- title: 'no filters'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: '*'
- },
- isNotEmptyFilter: false,
- title: 'empty filters'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name filter'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: '*'
- },
- isNotEmptyFilter: true,
- title: 'name filter with "0" as string'
- },
- {
- currentNameFilter: '',
- currentTypeFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'type filter'
- },
- {
- currentNameFilter: 'a',
- currentTypeFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'both filters'
- },
- {
- currentNameFilter: '0',
- currentTypeFilter: {
- value: false
- },
- isNotEmptyFilter: true,
- title: 'both filters with "0" as string'
- }
- ];
-
- cases.forEach(function (item) {
- it(item.title, function () {
- $httpBackend.expectGET(/\/api\/v1\/groups/).respond(200);
- scope.filter.name = item.currentNameFilter;
- scope.filter.type = item.currentTypeFilter;
- scope.$digest();
- expect(scope.isNotEmptyFilter).toEqual(item.isNotEmptyFilter);
- });
- });
-
- });
-
});
});
[19/51] [abbrv] ambari git commit: AMBARI-22571. Handle
passwords/sensitive data in Ambari configuration properties (Sandor Molnar
via rlevas)
Posted by am...@apache.org.
AMBARI-22571. Handle passwords/sensitive data in Ambari configuration properties (Sandor Molnar via rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/be0e87be
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/be0e87be
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/be0e87be
Branch: refs/heads/branch-feature-AMBARI-22008-isilon
Commit: be0e87bef52458955eddf1b824e6c5ddc6f31de6
Parents: afe469c
Author: Sandor Molnar <sm...@hortonworks.com>
Authored: Mon Dec 18 23:38:29 2017 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Mon Dec 18 23:38:29 2017 -0500
----------------------------------------------------------------------
ambari-server/pom.xml | 2 +-
.../RootServiceComponentConfiguration.java | 73 +++++++
.../server/configuration/Configuration.java | 11 +-
.../ConfigurationPropertyType.java | 22 +++
.../AmbariServerConfigurationHandler.java | 102 ++++++++--
.../AmbariServerConfigurationUtils.java | 78 ++++++++
.../AmbariServerLDAPConfigurationHandler.java | 9 +-
...ootServiceComponentConfigurationHandler.java | 7 +-
...eComponentConfigurationResourceProvider.java | 18 +-
.../domain/AmbariLdapConfigurationKeys.java | 77 ++++----
...ponentConfigurationResourceProviderTest.java | 192 +++++++++++++------
11 files changed, 473 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 2ecf435..5a0afb7 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1169,7 +1169,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.4</version>
+ <version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootServiceComponentConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootServiceComponentConfiguration.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootServiceComponentConfiguration.java
new file mode 100644
index 0000000..cd31c75
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootServiceComponentConfiguration.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+public class RootServiceComponentConfiguration {
+
+ private final Map<String, String> properties;
+ private final Map<String, String> propertyTypes;
+
+ public RootServiceComponentConfiguration() {
+ this(new TreeMap<>(), new TreeMap<>());
+ }
+
+ public RootServiceComponentConfiguration(Map<String, String> properties, Map<String, String> propertyTypes) {
+ this.properties = properties == null ? new TreeMap<>() : properties;
+ this.propertyTypes = propertyTypes == null ? new TreeMap<>() : propertyTypes;
+ }
+
+ public void addProperty(String propertyName, String propertyValue) {
+ properties.put(propertyName, propertyValue);
+ }
+
+ public void addPropertyType(String propertyName, String propertyType) {
+ propertyTypes.put(propertyName, propertyType);
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public Map<String, String> getPropertyTypes() {
+ return propertyTypes;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return EqualsBuilder.reflectionEquals(this, obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return HashCodeBuilder.reflectionHashCode(3, 19, this);
+ }
+
+ @Override
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index ea44a90..7003c98 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2783,6 +2783,9 @@ public class Configuration {
public static final ConfigurationProperty<String> DISPATCH_PROPERTY_SCRIPT_DIRECTORY = new ConfigurationProperty<>(
"notification.dispatch.alert.script.directory",AmbariPath.getPath("/var/lib/ambari-server/resources/scripts"));
+ @Markdown(description = "Whether security password encryption is enabled or not. In case it is we store passwords in their own file(s); otherwise we store passwords in the Ambari credential store.")
+ public static final ConfigurationProperty<Boolean> SECURITY_PASSWORD_ENCRYPTON_ENABLED = new ConfigurationProperty<Boolean>("security.passwords.encryption.enabled", false);
+
private static final Logger LOG = LoggerFactory.getLogger(
Configuration.class);
@@ -5693,6 +5696,13 @@ public class Configuration {
}
/**
+ * @return whether security password encryption is enabled or not (defaults to {@code false})
+ */
+ public boolean isSecurityPasswordEncryptionEnabled() {
+ return Boolean.parseBoolean(getProperty(SECURITY_PASSWORD_ENCRYPTON_ENABLED));
+ }
+
+ /**
* Generates a markdown table which includes:
* <ul>
* <li>Property key name</li>
@@ -5869,7 +5879,6 @@ public class Configuration {
private ConfigurationProperty(String key, T defaultValue) {
m_key = key;
m_defaultValue = defaultValue;
-
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/configuration/ConfigurationPropertyType.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/ConfigurationPropertyType.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/ConfigurationPropertyType.java
new file mode 100644
index 0000000..2e61c19
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/ConfigurationPropertyType.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.configuration;
+
+/**
+ * Constants representing types for AMBARI-level properties that are being stored in the DB
+ */
+public enum ConfigurationPropertyType {
+ PLAINTEXT, PASSWORD;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationHandler.java
index 34285d6..f3ff2dd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationHandler.java
@@ -18,23 +18,33 @@
package org.apache.ambari.server.controller.internal;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.StaticallyInject;
+import org.apache.ambari.server.api.services.RootServiceComponentConfiguration;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.events.AmbariConfigurationChangedEvent;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.orm.dao.AmbariConfigurationDAO;
import org.apache.ambari.server.orm.entities.AmbariConfigurationEntity;
+import org.apache.ambari.server.security.encryption.CredentialProvider;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
+
/**
* AmbariServerConfigurationHandler handles Ambari server specific configuration properties.
*/
@@ -48,11 +58,15 @@ class AmbariServerConfigurationHandler extends RootServiceComponentConfiguration
@Inject
private static AmbariEventPublisher publisher;
+ @Inject
+ private static Configuration ambariConfiguration;
+
+ private CredentialProvider credentialProvider;
@Override
- public Map<String, Map<String, String>> getConfigurations(String categoryName)
+ public Map<String, RootServiceComponentConfiguration> getConfigurations(String categoryName)
throws NoSuchResourceException {
- Map<String, Map<String, String>> configurations = null;
+ Map<String, RootServiceComponentConfiguration> configurations = null;
List<AmbariConfigurationEntity> entities = (categoryName == null)
? ambariConfigurationDAO.findAll()
@@ -60,17 +74,18 @@ class AmbariServerConfigurationHandler extends RootServiceComponentConfiguration
if (entities != null) {
configurations = new HashMap<>();
-
for (AmbariConfigurationEntity entity : entities) {
String category = entity.getCategoryName();
- Map<String, String> properties = configurations.get(category);
-
- if (properties == null) {
- properties = new TreeMap<>();
- configurations.put(category, properties);
+ RootServiceComponentConfiguration configuration = configurations.get(category);
+ if (configuration == null) {
+ configuration = new RootServiceComponentConfiguration();
+ configurations.put(category, configuration);
}
- properties.put(entity.getPropertyName(), entity.getPropertyValue());
+ configuration.addProperty(entity.getPropertyName(), entity.getPropertyValue());
+ if (categoryName != null) {
+ configuration.addPropertyType(entity.getPropertyName(), AmbariServerConfigurationUtils.getConfigurationPropertyTypeName(categoryName, entity.getPropertyName()));
+ }
}
}
@@ -94,13 +109,76 @@ class AmbariServerConfigurationHandler extends RootServiceComponentConfiguration
}
@Override
- public void updateCategory(String categoryName, Map<String, String> properties, boolean removePropertiesIfNotSpecified) {
- if (ambariConfigurationDAO.reconcileCategory(categoryName, properties, removePropertiesIfNotSpecified)) {
+ public void updateCategory(String categoryName, Map<String, String> properties, boolean removePropertiesIfNotSpecified) throws AmbariException {
+ boolean toBePublished = false;
+ final Iterator<Map.Entry<String, String>> propertiesIterator = properties.entrySet().iterator();
+ while (propertiesIterator.hasNext()) {
+ Map.Entry<String, String> property = propertiesIterator.next();
+ if (AmbariServerConfigurationUtils.isPassword(categoryName, property.getKey())) {
+ if (updatePasswordIfNeeded(categoryName, property.getKey(), property.getValue())) {
+ toBePublished = true;
+ }
+ propertiesIterator.remove(); //we do not need to change the any PASSWORD type configuration going forward
+ }
+ }
+
+ if (!properties.isEmpty()) {
+ toBePublished = ambariConfigurationDAO.reconcileCategory(categoryName, properties, removePropertiesIfNotSpecified) || toBePublished;
+ }
+
+ if (toBePublished) {
// notify subscribers about the configuration changes
publisher.publish(new AmbariConfigurationChangedEvent(categoryName));
}
}
+ private boolean updatePasswordIfNeeded(String categoryName, String propertyName, String newPassword) throws AmbariException {
+ if (newPassword != null) {
+ final String passwordFileOrCredentailStoreAlias = fetchPasswordFileNameOrCredentialStoreAlias(categoryName, propertyName);
+ if (!newPassword.equals(passwordFileOrCredentailStoreAlias)) { //we only need to do anything if the user-supplied password is a 'real' password
+ if (ambariConfiguration.isSecurityPasswordEncryptionEnabled()) {
+ getCredentialProvider().addAliasToCredentialStore(passwordFileOrCredentailStoreAlias, newPassword);
+ } else {
+ savePasswordInFile(passwordFileOrCredentailStoreAlias, newPassword);
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /*
+ * If the configuration element is actually a PASSWORD type element then we either have a password file name stored in the DB
+ * or - in case security password encryption is enabled - a Credential Store alias.
+ */
+ private String fetchPasswordFileNameOrCredentialStoreAlias(String categoryName, String propertyName) {
+ for (AmbariConfigurationEntity entity : ambariConfigurationDAO.findByCategory(categoryName)) {
+ if (entity.getPropertyName().equals(propertyName)) {
+ return entity.getPropertyValue();
+ }
+ }
+
+ return null;
+ }
+
+ private CredentialProvider getCredentialProvider() throws AmbariException {
+ if (credentialProvider == null) {
+ credentialProvider = new CredentialProvider(null, ambariConfiguration.getMasterKeyLocation(),
+ ambariConfiguration.isMasterKeyPersisted(), ambariConfiguration.getMasterKeyStoreLocation());
+ }
+ return credentialProvider;
+ }
+
+ private void savePasswordInFile(String passwordFileName, String newPassword) throws AmbariException {
+ try {
+ if (StringUtils.isNotBlank(passwordFileName)) {
+ FileUtils.writeStringToFile(new File(passwordFileName), newPassword, Charset.defaultCharset());
+ }
+ } catch (IOException e) {
+ throw new AmbariException("Error while updating password file [" + passwordFileName + "]", e);
+ }
+ }
+
@Override
public OperationResult performOperation(String categoryName, Map<String, String> properties,
boolean mergeExistingProperties, String operation,
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationUtils.java
new file mode 100644
index 0000000..ac05f93
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationUtils.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import org.apache.ambari.server.configuration.ConfigurationPropertyType;
+import org.apache.ambari.server.ldap.domain.AmbariLdapConfigurationKeys;
+
+/**
+ * Provides useful utility methods for AMBARI-level configuration related tasks.
+ */
+public class AmbariServerConfigurationUtils {
+
+ /**
+ * @param category
+ * the name of the category
+ * @param propertyName
+ * the name of the property
+ * @return the type of the given category/property if such category/property
+ * exists; {@code null} otherwise
+ * @throws IllegalStateException
+ * if there is no property found with the given name
+ */
+ public static ConfigurationPropertyType getConfigurationPropertyType(String category, String propertyName) {
+ if (AmbariServerConfigurationCategory.LDAP_CONFIGURATION.getCategoryName().equals(category)) {
+ return AmbariLdapConfigurationKeys.fromKeyStr(propertyName).getConfigurationPropertyType();
+ }
+ return null;
+ }
+
+ /**
+ * @param category
+ * the name of the category
+ * @param propertyName
+ * the name of the property
+ * @return the String representation of the type if such category/property
+ * exists; {@code null} otherwise * @throws IllegalStateException if
+ * there is no property found with the given name
+ */
+ public static String getConfigurationPropertyTypeName(String category, String propertyName) {
+ final ConfigurationPropertyType configurationPropertyType = getConfigurationPropertyType(category, propertyName);
+ return configurationPropertyType == null ? null : configurationPropertyType.name();
+ }
+
+ /**
+ * Indicates whether the given property's type is
+ *
+ * {@link ConfigurationPropertyType.PASSWORD}
+ *
+ * @param category
+ * the name of the category
+ * @param propertyName
+ * the name of the property
+ * @return {@code true} in case the given property's type is
+ * {@link ConfigurationPropertyType.PASSWORD}; {@code false} otherwise
+ * @throws IllegalStateException
+ * if there is no property found with the given name
+ */
+ public static boolean isPassword(String category, String propertyName) {
+ return ConfigurationPropertyType.PASSWORD.equals(getConfigurationPropertyType(category, propertyName));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
index 6f16c49..c4dca80 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.StaticallyInject;
+import org.apache.ambari.server.api.services.RootServiceComponentConfiguration;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.ldap.domain.AmbariLdapConfiguration;
@@ -44,7 +45,7 @@ class AmbariServerLDAPConfigurationHandler extends AmbariServerConfigurationHand
@Inject
private static LdapFacade ldapFacade;
-
+
@Override
public OperationResult performOperation(String categoryName, Map<String, String> properties,
boolean mergeExistingProperties, String operation, Map<String, Object> operationParameters) throws SystemException {
@@ -64,12 +65,12 @@ class AmbariServerLDAPConfigurationHandler extends AmbariServerConfigurationHand
// to retrieve if. If one does not exist, that is ok.
if (mergeExistingProperties) {
try {
- Map<String, Map<String, String>> _configurations = getConfigurations(categoryName);
+ Map<String, RootServiceComponentConfiguration> _configurations = getConfigurations(categoryName);
if (_configurations != null) {
- Map<String, String> _ldapProperties = _configurations.get(categoryName);
+ RootServiceComponentConfiguration _ldapProperties = _configurations.get(categoryName);
if (_ldapProperties != null) {
- ldapConfigurationProperties.putAll(_ldapProperties);
+ ldapConfigurationProperties.putAll(_ldapProperties.getProperties());
}
}
} catch (NoSuchResourceException e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationHandler.java
index 3b58ce1..e9e4c9e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationHandler.java
@@ -20,6 +20,8 @@ package org.apache.ambari.server.controller.internal;
import java.util.Map;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.RootServiceComponentConfiguration;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.SystemException;
@@ -35,7 +37,7 @@ abstract class RootServiceComponentConfigurationHandler {
* @return a map of category names to properties (name/value pairs).
* @throws NoSuchResourceException if the requested data is not found
*/
- public abstract Map<String, Map<String, String>> getConfigurations(String categoryName) throws NoSuchResourceException;
+ public abstract Map<String, RootServiceComponentConfiguration> getConfigurations(String categoryName) throws NoSuchResourceException;
/**
* Delete the requested configuration.
@@ -58,8 +60,9 @@ abstract class RootServiceComponentConfigurationHandler {
* @param properties a map of properties to set
* @param removePropertiesIfNotSpecified <code>true</code> to ensure the set of properties are only those that have be explicitly specified;
* <code>false</code> to update the set of existing properties with the specified set of properties, adding missing properties but not removing any properties
+ * @throws AmbariException in case an error occurred while updating category's properties
*/
- public abstract void updateCategory(String categoryName, Map<String, String> properties, boolean removePropertiesIfNotSpecified);
+ public abstract void updateCategory(String categoryName, Map<String, String> properties, boolean removePropertiesIfNotSpecified) throws AmbariException;
/**
* Preform some operation on the set of data for a category.
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProvider.java
index 74f8a4d..543cf94 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProvider.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.RootServiceComponentConfiguration;
import org.apache.ambari.server.api.services.RootServiceComponentConfigurationService;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
@@ -50,6 +51,7 @@ public class RootServiceComponentConfigurationResourceProvider extends AbstractA
public static final String CONFIGURATION_CATEGORY_PROPERTY_ID = PropertyHelper.getPropertyId(RESOURCE_KEY, "category");
public static final String CONFIGURATION_PROPERTIES_PROPERTY_ID = PropertyHelper.getPropertyId(RESOURCE_KEY, "properties");
+ public static final String CONFIGURATION_PROPERTY_TYPES_PROPERTY_ID = PropertyHelper.getPropertyId(RESOURCE_KEY, "property_types");
public static final String CONFIGURATION_COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId(RESOURCE_KEY, "component_name");
public static final String CONFIGURATION_SERVICE_NAME_PROPERTY_ID = PropertyHelper.getPropertyId(RESOURCE_KEY, "service_name");
@@ -65,6 +67,7 @@ public class RootServiceComponentConfigurationResourceProvider extends AbstractA
set.add(CONFIGURATION_COMPONENT_NAME_PROPERTY_ID);
set.add(CONFIGURATION_CATEGORY_PROPERTY_ID);
set.add(CONFIGURATION_PROPERTIES_PROPERTY_ID);
+ set.add(CONFIGURATION_PROPERTY_TYPES_PROPERTY_ID);
PROPERTIES = Collections.unmodifiableSet(set);
@@ -202,12 +205,13 @@ public class RootServiceComponentConfigurationResourceProvider extends AbstractA
return getRequestStatus(null, null, operationStatusMetadata);
}
- private Resource toResource(String serviceName, String componentName, String categoryName, Map<String, String> properties, Set<String> requestedIds) {
+ private Resource toResource(String serviceName, String componentName, String categoryName, Map<String, String> properties, Map<String, String> propertyTypes, Set<String> requestedIds) {
Resource resource = new ResourceImpl(Resource.Type.RootServiceComponentConfiguration);
setResourceProperty(resource, CONFIGURATION_SERVICE_NAME_PROPERTY_ID, serviceName, requestedIds);
setResourceProperty(resource, CONFIGURATION_COMPONENT_NAME_PROPERTY_ID, componentName, requestedIds);
setResourceProperty(resource, CONFIGURATION_CATEGORY_PROPERTY_ID, categoryName, requestedIds);
setResourceProperty(resource, CONFIGURATION_PROPERTIES_PROPERTY_ID, properties, requestedIds);
+ setResourceProperty(resource, CONFIGURATION_PROPERTY_TYPES_PROPERTY_ID, propertyTypes, requestedIds);
return resource;
}
@@ -238,7 +242,11 @@ public class RootServiceComponentConfigurationResourceProvider extends AbstractA
RootServiceComponentConfigurationHandler handler = rootServiceComponentConfigurationHandlerFactory.getInstance(requestDetails.serviceName, requestDetails.componentName, requestDetails.categoryName);
if (handler != null) {
- handler.updateCategory(requestDetails.categoryName, requestDetails.properties, removePropertiesIfNotSpecified);
+ try {
+ handler.updateCategory(requestDetails.categoryName, requestDetails.properties, removePropertiesIfNotSpecified);
+ } catch (AmbariException e) {
+ throw new SystemException(e.getMessage(), e.getCause());
+ }
} else {
throw new SystemException(String.format("Configurations may not be updated for the %s component of the root service, %s", requestDetails.serviceName, requestDetails.componentName));
}
@@ -390,11 +398,11 @@ public class RootServiceComponentConfigurationResourceProvider extends AbstractA
RootServiceComponentConfigurationHandler handler = rootServiceComponentConfigurationHandlerFactory.getInstance(serviceName, componentName, categoryName);
if (handler != null) {
- Map<String, Map<String, String>> configurations = handler.getConfigurations(categoryName);
+ Map<String, RootServiceComponentConfiguration> configurations = handler.getConfigurations(categoryName);
if (configurations != null) {
- for (Map.Entry<String, Map<String, String>> entry : configurations.entrySet()) {
- resources.add(toResource(serviceName, componentName, entry.getKey(), entry.getValue(), requestedIds));
+ for (Map.Entry<String, RootServiceComponentConfiguration> entry : configurations.entrySet()) {
+ resources.add(toResource(serviceName, componentName, entry.getKey(), entry.getValue().getProperties(), entry.getValue().getPropertyTypes(), requestedIds));
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfigurationKeys.java b/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfigurationKeys.java
index 2e1c36b..b7a713b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfigurationKeys.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfigurationKeys.java
@@ -14,62 +14,73 @@
package org.apache.ambari.server.ldap.domain;
+import static org.apache.ambari.server.configuration.ConfigurationPropertyType.PASSWORD;
+import static org.apache.ambari.server.configuration.ConfigurationPropertyType.PLAINTEXT;
+
+import org.apache.ambari.server.configuration.ConfigurationPropertyType;
+
/**
* Constants representing supported LDAP related property names
- * // todo extend this with validation information, description, defaults maybe
+ * // TODO: extend this with validation information, description, defaults maybe
*/
public enum AmbariLdapConfigurationKeys {
- LDAP_ENABLED("ambari.ldap.authentication.enabled"),
- SERVER_HOST("ambari.ldap.connectivity.server.host"),
- SERVER_PORT("ambari.ldap.connectivity.server.port"),
- USE_SSL("ambari.ldap.connectivity.use_ssl"),
+ LDAP_ENABLED("ambari.ldap.authentication.enabled", PLAINTEXT),
+ SERVER_HOST("ambari.ldap.connectivity.server.host", PLAINTEXT),
+ SERVER_PORT("ambari.ldap.connectivity.server.port", PLAINTEXT),
+ USE_SSL("ambari.ldap.connectivity.use_ssl", PLAINTEXT),
- TRUST_STORE("ambari.ldap.connectivity.trust_store"),
- TRUST_STORE_TYPE("ambari.ldap.connectivity.trust_store.type"),
- TRUST_STORE_PATH("ambari.ldap.connectivity.trust_store.path"),
- TRUST_STORE_PASSWORD("ambari.ldap.connectivity.trust_store.password"),
- ANONYMOUS_BIND("ambari.ldap.connectivity.anonymous_bind"),
+ TRUST_STORE("ambari.ldap.connectivity.trust_store", PLAINTEXT),
+ TRUST_STORE_TYPE("ambari.ldap.connectivity.trust_store.type", PLAINTEXT),
+ TRUST_STORE_PATH("ambari.ldap.connectivity.trust_store.path", PLAINTEXT),
+ TRUST_STORE_PASSWORD("ambari.ldap.connectivity.trust_store.password", PASSWORD),
+ ANONYMOUS_BIND("ambari.ldap.connectivity.anonymous_bind", PLAINTEXT),
- BIND_DN("ambari.ldap.connectivity.bind_dn"),
- BIND_PASSWORD("ambari.ldap.connectivity.bind_password"),
+ BIND_DN("ambari.ldap.connectivity.bind_dn", PLAINTEXT),
+ BIND_PASSWORD("ambari.ldap.connectivity.bind_password", PASSWORD),
- ATTR_DETECTION("ambari.ldap.attributes.detection"), // manual | auto
+ ATTR_DETECTION("ambari.ldap.attributes.detection", PLAINTEXT), // manual | auto
- DN_ATTRIBUTE("ambari.ldap.attributes.dn_attr"),
+ DN_ATTRIBUTE("ambari.ldap.attributes.dn_attr", PLAINTEXT),
- USER_OBJECT_CLASS("ambari.ldap.attributes.user.object_class"),
- USER_NAME_ATTRIBUTE("ambari.ldap.attributes.user.name_attr"),
- USER_GROUP_MEMBER_ATTRIBUTE("ambari.ldap.attributes.user.group_member_attr"),
- USER_SEARCH_BASE("ambari.ldap.attributes.user.search_base"),
+ USER_OBJECT_CLASS("ambari.ldap.attributes.user.object_class", PLAINTEXT),
+ USER_NAME_ATTRIBUTE("ambari.ldap.attributes.user.name_attr", PLAINTEXT),
+ USER_GROUP_MEMBER_ATTRIBUTE("ambari.ldap.attributes.user.group_member_attr", PLAINTEXT),
+ USER_SEARCH_BASE("ambari.ldap.attributes.user.search_base", PLAINTEXT),
- GROUP_OBJECT_CLASS("ambari.ldap.attributes.group.object_class"),
- GROUP_NAME_ATTRIBUTE("ambari.ldap.attributes.group.name_attr"),
- GROUP_MEMBER_ATTRIBUTE("ambari.ldap.attributes.group.member_attr"),
- GROUP_SEARCH_BASE("ambari.ldap.attributes.group.search_base"),
+ GROUP_OBJECT_CLASS("ambari.ldap.attributes.group.object_class", PLAINTEXT),
+ GROUP_NAME_ATTRIBUTE("ambari.ldap.attributes.group.name_attr", PLAINTEXT),
+ GROUP_MEMBER_ATTRIBUTE("ambari.ldap.attributes.group.member_attr", PLAINTEXT),
+ GROUP_SEARCH_BASE("ambari.ldap.attributes.group.search_base", PLAINTEXT),
- USER_SEARCH_FILTER("ambari.ldap.advanced.user_search_filter"),
- USER_MEMBER_REPLACE_PATTERN("ambari.ldap.advanced.user_member_replace_pattern"),
- USER_MEMBER_FILTER("ambari.ldap.advanced.user_member_filter"),
+ USER_SEARCH_FILTER("ambari.ldap.advanced.user_search_filter", PLAINTEXT),
+ USER_MEMBER_REPLACE_PATTERN("ambari.ldap.advanced.user_member_replace_pattern", PLAINTEXT),
+ USER_MEMBER_FILTER("ambari.ldap.advanced.user_member_filter", PLAINTEXT),
- GROUP_SEARCH_FILTER("ambari.ldap.advanced.group_search_filter"),
- GROUP_MEMBER_REPLACE_PATTERN("ambari.ldap.advanced.group_member_replace_pattern"),
- GROUP_MEMBER_FILTER("ambari.ldap.advanced.group_member_filter"),
+ GROUP_SEARCH_FILTER("ambari.ldap.advanced.group_search_filter", PLAINTEXT),
+ GROUP_MEMBER_REPLACE_PATTERN("ambari.ldap.advanced.group_member_replace_pattern", PLAINTEXT),
+ GROUP_MEMBER_FILTER("ambari.ldap.advanced.group_member_filter", PLAINTEXT),
- FORCE_LOWERCASE_USERNAMES("ambari.ldap.advanced.force_lowercase_usernames"),
- REFERRAL_HANDLING("ambari.ldap.advanced.referrals"), // folow
- PAGINATION_ENABLED("ambari.ldap.advanced.pagination_enabled"); // true | false
+ FORCE_LOWERCASE_USERNAMES("ambari.ldap.advanced.force_lowercase_usernames", PLAINTEXT),
+ REFERRAL_HANDLING("ambari.ldap.advanced.referrals", PLAINTEXT), // folow
+ PAGINATION_ENABLED("ambari.ldap.advanced.pagination_enabled", PLAINTEXT); // true | false
private String propertyName;
+ private ConfigurationPropertyType configurationPropertyType;
- AmbariLdapConfigurationKeys(String propName) {
+ AmbariLdapConfigurationKeys(String propName, ConfigurationPropertyType configurationPropertyType) {
this.propertyName = propName;
+ this.configurationPropertyType = configurationPropertyType;
}
public String key() {
return this.propertyName;
}
+ public ConfigurationPropertyType getConfigurationPropertyType() {
+ return configurationPropertyType;
+ }
+
public static AmbariLdapConfigurationKeys fromKeyStr(String keyStr) {
for (AmbariLdapConfigurationKeys key : values()) {
if (key.key().equals(keyStr)) {
@@ -77,7 +88,7 @@ public enum AmbariLdapConfigurationKeys {
}
}
- throw new IllegalStateException("invalid konfiguration key found!");
+ throw new IllegalStateException("invalid configuration key found!");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be0e87be/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProviderTest.java
index 13c644a..bd0ff1d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RootServiceComponentConfigurationResourceProviderTest.java
@@ -25,6 +25,8 @@ import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.newCapture;
+import java.io.File;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -36,6 +38,7 @@ import java.util.TreeMap;
import javax.persistence.EntityManager;
import org.apache.ambari.server.api.services.RootServiceComponentConfigurationService;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.RootComponent;
import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.controller.predicate.AndPredicate;
@@ -47,14 +50,23 @@ import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.apache.ambari.server.events.AmbariConfigurationChangedEvent;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
+import org.apache.ambari.server.ldap.domain.AmbariLdapConfigurationKeys;
import org.apache.ambari.server.orm.dao.AmbariConfigurationDAO;
import org.apache.ambari.server.orm.entities.AmbariConfigurationEntity;
import org.apache.ambari.server.security.TestAuthenticationFactory;
import org.apache.ambari.server.security.authorization.AuthorizationException;
+import org.apache.ambari.server.security.encryption.CredentialProvider;
+import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.commons.io.FileUtils;
import org.easymock.Capture;
import org.easymock.EasyMockSupport;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
@@ -64,11 +76,40 @@ import com.google.inject.Injector;
import edu.emory.mathcs.backport.java.util.Collections;
import junit.framework.Assert;
+import junit.framework.AssertionFailedError;
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({FileUtils.class, AmbariServerConfigurationHandler.class})
public class RootServiceComponentConfigurationResourceProviderTest extends EasyMockSupport {
private static final String CATEGORY_NAME_1 = "test-category-1";
private static final String CATEGORY_NAME_2 = "test-category-2";
+ private static final String LDAP_CONFIG_CATEGORY = AmbariServerConfigurationCategory.LDAP_CONFIGURATION.getCategoryName();
+
+ private Injector injector;
+ private Predicate predicate;
+ private ResourceProvider resourceProvider;
+ private RootServiceComponentConfigurationHandlerFactory factory;
+ private Request request;
+ private AmbariConfigurationDAO dao;
+ private Configuration configuration;
+ private AmbariEventPublisher publisher;
+ private Map<String, String> properties = new HashMap<>();
+ private Set<Map<String, Object>> propertySets = new HashSet<>();
+
+ @Before
+ public void init() throws Exception {
+ injector = createInjector();
+ resourceProvider = injector.getInstance(RootServiceComponentConfigurationResourceProvider.class);
+ predicate = createPredicate(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1);
+ request = createMock(Request.class);
+ dao = injector.getInstance(AmbariConfigurationDAO.class);
+ configuration = injector.getInstance(Configuration.class);
+ factory = injector.getInstance(RootServiceComponentConfigurationHandlerFactory.class);
+ publisher = injector.getInstance(AmbariEventPublisher.class);
+ properties = new HashMap<>();
+ propertySets = new HashSet<>();
+ }
@After
public void clearAuthentication() {
@@ -126,21 +167,14 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
}
private void testCreateResources(Authentication authentication, String opDirective) throws Exception {
- Injector injector = createInjector();
-
- ResourceProvider resourceProvider = injector.getInstance(RootServiceComponentConfigurationResourceProvider.class);
-
- Set<Map<String, Object>> propertySets = new HashSet<>();
-
- Map<String, String> properties1 = new HashMap<>();
- properties1.put("property1a", "value1");
- properties1.put("property2a", "value2");
- propertySets.add(toRequestProperties(CATEGORY_NAME_1, properties1));
+ properties.put(Configuration.AMBARI_PYTHON_WRAP.getKey(), "value1");
+ properties.put(Configuration.AMBARI_DISPLAY_URL.getKey(), "value2");
+ propertySets.add(toRequestProperties(CATEGORY_NAME_1, properties));
Map<String, String> properties2 = new HashMap<>();
if (opDirective == null) {
- properties2.put("property1b", "value1");
- properties2.put("property2b", "value2");
+ properties2.put(Configuration.SSL_TRUSTSTORE_TYPE.getKey(), "value1");
+ properties2.put(Configuration.SSL_TRUSTSTORE_PATH.getKey(), "value2");
propertySets.add(toRequestProperties(CATEGORY_NAME_2, properties2));
}
@@ -151,7 +185,6 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
requestInfoProperties = Collections.singletonMap(RootServiceComponentConfigurationService.DIRECTIVE_OPERATION, opDirective);
}
- Request request = createMock(Request.class);
expect(request.getProperties()).andReturn(propertySets).once();
expect(request.getRequestInfoProperties()).andReturn(requestInfoProperties).once();
@@ -159,7 +192,6 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
Capture<Map<String, String>> capturedProperties2 = newCapture();
if (opDirective == null) {
- AmbariConfigurationDAO dao = injector.getInstance(AmbariConfigurationDAO.class);
expect(dao.reconcileCategory(eq(CATEGORY_NAME_1), capture(capturedProperties1), eq(true)))
.andReturn(true)
.once();
@@ -168,12 +200,10 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
.once();
- AmbariEventPublisher publisher = injector.getInstance(AmbariEventPublisher.class);
publisher.publish(anyObject(AmbariConfigurationChangedEvent.class));
expectLastCall().times(2);
}
- RootServiceComponentConfigurationHandlerFactory factory = injector.getInstance(RootServiceComponentConfigurationHandlerFactory.class);
expect(factory.getInstance(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1))
.andReturn(new AmbariServerConfigurationHandler())
.once();
@@ -205,7 +235,7 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
verifyAll();
if (opDirective == null) {
- validateCapturedProperties(properties1, capturedProperties1);
+ validateCapturedProperties(properties, capturedProperties1);
validateCapturedProperties(properties2, capturedProperties2);
} else {
Assert.assertFalse(capturedProperties1.hasCaptured());
@@ -239,22 +269,11 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
}
private void testDeleteResources(Authentication authentication) throws Exception {
- Injector injector = createInjector();
-
- ResourceProvider resourceProvider = injector.getInstance(RootServiceComponentConfigurationResourceProvider.class);
-
- Predicate predicate = createPredicate(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1);
-
- Request request = createMock(Request.class);
-
- AmbariConfigurationDAO dao = injector.getInstance(AmbariConfigurationDAO.class);
expect(dao.removeByCategory(CATEGORY_NAME_1)).andReturn(1).once();
- AmbariEventPublisher publisher = injector.getInstance(AmbariEventPublisher.class);
publisher.publish(anyObject(AmbariConfigurationChangedEvent.class));
expectLastCall().once();
- RootServiceComponentConfigurationHandlerFactory factory = injector.getInstance(RootServiceComponentConfigurationHandlerFactory.class);
expect(factory.getInstance(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1))
.andReturn(new AmbariServerConfigurationHandler())
.once();
@@ -294,25 +313,15 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
}
private void testGetResources(Authentication authentication) throws Exception {
- Injector injector = createInjector();
-
- ResourceProvider resourceProvider = injector.getInstance(RootServiceComponentConfigurationResourceProvider.class);
-
- Predicate predicate = createPredicate(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1);
-
- Request request = createMock(Request.class);
expect(request.getPropertyIds()).andReturn(null).anyTimes();
- Map<String, String> properties = new HashMap<>();
- properties.put("property1a", "value1");
- properties.put("property2a", "value2");
+ properties.put(AmbariLdapConfigurationKeys.ANONYMOUS_BIND.key(), "value1");
+ properties.put(AmbariLdapConfigurationKeys.GROUP_MEMBER_ATTRIBUTE.key(), "value2");
- AmbariConfigurationDAO dao = injector.getInstance(AmbariConfigurationDAO.class);
expect(dao.findByCategory(CATEGORY_NAME_1)).andReturn(createEntities(CATEGORY_NAME_1, properties)).once();
- RootServiceComponentConfigurationHandlerFactory factory = injector.getInstance(RootServiceComponentConfigurationHandlerFactory.class);
expect(factory.getInstance(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1))
- .andReturn(new AmbariServerConfigurationHandler())
+ .andReturn(new AmbariServerLDAPConfigurationHandler())
.once();
replayAll();
@@ -330,16 +339,19 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
Assert.assertEquals(Resource.Type.RootServiceComponentConfiguration, resource.getType());
Map<String, Map<String, Object>> propertiesMap = resource.getPropertiesMap();
- Assert.assertEquals(2, propertiesMap.size());
+ Assert.assertEquals(3, propertiesMap.size());
Assert.assertEquals(CATEGORY_NAME_1, propertiesMap.get(RootServiceComponentConfigurationResourceProvider.RESOURCE_KEY).get("category"));
- Map<String, Object> retrievedProperties = propertiesMap.get(RootServiceComponentConfigurationResourceProvider.RESOURCE_KEY + "/properties");
+ Map<String, Object> retrievedProperties = propertiesMap.get(RootServiceComponentConfigurationResourceProvider.CONFIGURATION_PROPERTIES_PROPERTY_ID);
Assert.assertEquals(2, retrievedProperties.size());
for (Map.Entry<String, String> entry : properties.entrySet()) {
Assert.assertEquals(entry.getValue(), retrievedProperties.get(entry.getKey()));
}
+
+ Map<String, Object> retrievedPropertyTypes = propertiesMap.get(RootServiceComponentConfigurationResourceProvider.CONFIGURATION_PROPERTY_TYPES_PROPERTY_ID);
+ Assert.assertEquals(2, retrievedPropertyTypes.size());
}
@Test
@@ -393,18 +405,9 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
}
private void testUpdateResources(Authentication authentication, String opDirective) throws Exception {
- Injector injector = createInjector();
-
- ResourceProvider resourceProvider = injector.getInstance(RootServiceComponentConfigurationResourceProvider.class);
-
- Predicate predicate = createPredicate(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1);
-
- Set<Map<String, Object>> propertySets = new HashSet<>();
-
- Map<String, String> properties1 = new HashMap<>();
- properties1.put("property1a", "value1");
- properties1.put("property2a", "value2");
- propertySets.add(toRequestProperties(CATEGORY_NAME_1, properties1));
+ properties.put(Configuration.AMBARI_DISPLAY_URL.getKey(), "value1");
+ properties.put(Configuration.BOOTSTRAP_MASTER_HOSTNAME.getKey(), "value2");
+ propertySets.add(toRequestProperties(CATEGORY_NAME_1, properties));
Map<String, String> requestInfoProperties;
if (opDirective == null) {
@@ -413,24 +416,20 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
requestInfoProperties = Collections.singletonMap(RootServiceComponentConfigurationService.DIRECTIVE_OPERATION, opDirective);
}
- Request request = createMock(Request.class);
expect(request.getProperties()).andReturn(propertySets).once();
expect(request.getRequestInfoProperties()).andReturn(requestInfoProperties).once();
Capture<Map<String, String>> capturedProperties1 = newCapture();
if (opDirective == null) {
- AmbariConfigurationDAO dao = injector.getInstance(AmbariConfigurationDAO.class);
expect(dao.reconcileCategory(eq(CATEGORY_NAME_1), capture(capturedProperties1), eq(false)))
.andReturn(true)
.once();
- AmbariEventPublisher publisher = injector.getInstance(AmbariEventPublisher.class);
publisher.publish(anyObject(AmbariConfigurationChangedEvent.class));
expectLastCall().times(1);
}
- RootServiceComponentConfigurationHandlerFactory factory = injector.getInstance(RootServiceComponentConfigurationHandlerFactory.class);
expect(factory.getInstance(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), CATEGORY_NAME_1))
.andReturn(new AmbariServerConfigurationHandler())
.once();
@@ -458,12 +457,83 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
verifyAll();
if (opDirective == null) {
- validateCapturedProperties(properties1, capturedProperties1);
+ validateCapturedProperties(properties, capturedProperties1);
} else {
Assert.assertFalse(capturedProperties1.hasCaptured());
}
}
+ @Test
+ public void shouldNotUpdatePasswordIfItHasNotBeenChanged() throws Exception {
+ SecurityContextHolder.getContext().setAuthentication(TestAuthenticationFactory.createAdministrator());
+ properties.put(AmbariLdapConfigurationKeys.BIND_PASSWORD.key(), "passwd");
+ propertySets.add(toRequestProperties(LDAP_CONFIG_CATEGORY, properties));
+ setupBasicExpectations(properties);
+ expect(configuration.isSecurityPasswordEncryptionEnabled()).andThrow(new AssertionFailedError()).anyTimes(); //this call should never have never been hit
+
+ replayAll();
+ resourceProvider.updateResources(request, predicate);
+ verifyAll();
+ }
+
+ @Test
+ public void shouldUpdatePasswordFileIfSecurityPasswordEncryptionIsDisabled() throws Exception {
+ SecurityContextHolder.getContext().setAuthentication(TestAuthenticationFactory.createAdministrator());
+ properties.put(AmbariLdapConfigurationKeys.BIND_PASSWORD.key(), "newPasswd");
+ propertySets.add(toRequestProperties(LDAP_CONFIG_CATEGORY, properties));
+ Map<String, String> expectedProperties = new HashMap<>();
+ expectedProperties.put(AmbariLdapConfigurationKeys.BIND_PASSWORD.key(), "currentPasswd");
+ setupBasicExpectations(expectedProperties);
+ expect(configuration.isSecurityPasswordEncryptionEnabled()).andReturn(false).once();
+ PowerMock.mockStatic(FileUtils.class);
+ FileUtils.writeStringToFile(new File("currentPasswd"), "newPasswd", Charset.defaultCharset());
+ PowerMock.expectLastCall().once();
+ PowerMock.replay(FileUtils.class);
+ publisher.publish(anyObject(AmbariConfigurationChangedEvent.class));
+ expectLastCall().once();
+
+ replayAll();
+ resourceProvider.updateResources(request, predicate);
+ verifyAll();
+ }
+
+ @Test
+ public void shouldUpdatePasswordInCredentialStoreIfSecurityPasswordEncryptionIsEnabled() throws Exception {
+ SecurityContextHolder.getContext().setAuthentication(TestAuthenticationFactory.createAdministrator());
+ properties.put(AmbariLdapConfigurationKeys.BIND_PASSWORD.key(), "newPasswd");
+ propertySets.add(toRequestProperties(LDAP_CONFIG_CATEGORY, properties));
+ Map<String, String> expectedProperties = new HashMap<>();
+ expectedProperties.put(AmbariLdapConfigurationKeys.BIND_PASSWORD.key(), "currentPasswd");
+ setupBasicExpectations(expectedProperties);
+ expect(configuration.isSecurityPasswordEncryptionEnabled()).andReturn(true).once();
+
+ File masterKeyLocation = createNiceMock(File.class);
+ File masterKeyStoreLocation = createNiceMock(File.class);
+ expect(configuration.getMasterKeyLocation()).andReturn(masterKeyLocation).once();
+ expect(configuration.isMasterKeyPersisted()).andReturn(false).once();
+ expect(configuration.getMasterKeyStoreLocation()).andReturn(masterKeyStoreLocation).once();
+ CredentialProvider credentialProvider = PowerMock.createMock(CredentialProvider.class);
+ PowerMock.expectNew(CredentialProvider.class, null, (String) null, masterKeyLocation, false, masterKeyStoreLocation).andReturn(credentialProvider);
+ credentialProvider.addAliasToCredentialStore("currentPasswd", "newPasswd");
+ PowerMock.expectLastCall().once();
+ PowerMock.replay(credentialProvider, CredentialProvider.class);
+
+ publisher.publish(anyObject(AmbariConfigurationChangedEvent.class));
+ expectLastCall().once();
+
+ replayAll();
+ resourceProvider.updateResources(request, predicate);
+ verifyAll();
+ PowerMock.verify(credentialProvider, CredentialProvider.class);
+ }
+
+ private void setupBasicExpectations(Map<String, String> expectedProperties) {
+ expect(request.getProperties()).andReturn(propertySets).once();
+ expect(request.getRequestInfoProperties()).andReturn(new HashMap<>());
+ expect(dao.findByCategory(LDAP_CONFIG_CATEGORY)).andReturn(createEntities(AmbariServerConfigurationCategory.LDAP_CONFIGURATION.getCategoryName(), expectedProperties)).once();
+ expect(factory.getInstance(RootService.AMBARI.name(), RootComponent.AMBARI_SERVER.name(), LDAP_CONFIG_CATEGORY)).andReturn(new AmbariServerLDAPConfigurationHandler()).once();
+ }
+
private Predicate createPredicate(String serviceName, String componentName, String categoryName) {
Predicate predicateService = new PredicateBuilder()
.property(CONFIGURATION_SERVICE_NAME_PROPERTY_ID)
@@ -521,6 +591,8 @@ public class RootServiceComponentConfigurationResourceProviderTest extends EasyM
return Guice.createInjector(new AbstractModule() {
@Override
protected void configure() {
+ bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+ bind(Configuration.class).toInstance(createNiceMock(Configuration.class));
bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class));
bind(AmbariConfigurationDAO.class).toInstance(createMock(AmbariConfigurationDAO.class));
bind(AmbariEventPublisher.class).toInstance(createMock(AmbariEventPublisher.class));
[36/51] [abbrv] ambari git commit: AMBARI-22514,
AMBARI-22653. Ambari Infra Manager: solr data exporting jobs and
integration test environment. (Krisztian Kasa via swagle)
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentSource.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentSource.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentSource.java
index 2181ba3..5ded9ac 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentSource.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrDocumentSource.java
@@ -18,6 +18,8 @@
*/
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.CloseableIterator;
+import org.apache.ambari.infra.job.ObjectSource;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -29,27 +31,29 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.time.format.DateTimeFormatter;
-public class SolrDocumentSource implements DocumentSource {
+public class SolrDocumentSource implements ObjectSource<Document> {
public static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");
private static final Logger LOG = LoggerFactory.getLogger(SolrDocumentSource.class);
- private final String zkHost;
+ private final String zkConnectionString;
private final SolrQueryProperties properties;
- private final String endValue;
+ private final String start;
+ private final String end;
- public SolrDocumentSource(String zkHost, SolrQueryProperties properties, String endValue) {
- this.zkHost = zkHost;
+ public SolrDocumentSource(String zkConnectionString, SolrQueryProperties properties, String start, String end) {
+ this.zkConnectionString = zkConnectionString;
this.properties = properties;
- this.endValue = endValue;
+ this.start = start;
+ this.end = end;
}
@Override
- public DocumentIterator open(Document current, int rows) {
- CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build();
+ public CloseableIterator<Document> open(Document current, int rows) {
+ CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkConnectionString).build();
client.setDefaultCollection(properties.getCollection());
SolrQuery query = properties.toQueryBuilder()
- .setEndValue(endValue)
+ .setInterval(start, end)
.setDocument(current)
.build();
query.setRows(rows);
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
index d0f6d40..b3ea14e 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
@@ -25,6 +25,7 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.solr.client.solrj.SolrQuery.ORDER.asc;
public class SolrQueryBuilder {
@@ -32,6 +33,7 @@ public class SolrQueryBuilder {
public static final Pattern PARAMETER_PATTERN = Pattern.compile("\\$\\{[a-z]+\\}");
private String queryText;
+ private String startValue;
private String endValue;
private String filterQueryText;
private Document document;
@@ -51,6 +53,12 @@ public class SolrQueryBuilder {
return this;
}
+ public SolrQueryBuilder setInterval(String startValue, String endValue) {
+ this.startValue = startValue;
+ this.endValue = endValue;
+ return this;
+ }
+
public SolrQueryBuilder setFilterQueryText(String filterQueryText) {
this.filterQueryText = filterQueryText;
return this;
@@ -71,19 +79,21 @@ public class SolrQueryBuilder {
SolrQuery solrQuery = new SolrQuery();
String query = queryText;
- query = setEndValueOn(query);
+ query = setValueOn(query, "${start}", startValue);
+ query = setValueOn(query, "${end}", endValue);
solrQuery.setQuery(query);
if (filterQueryText != null) {
String filterQuery = filterQueryText;
- filterQuery = setEndValueOn(filterQuery);
+ filterQuery = setValueOn(filterQuery, "${start}", startValue);
+ filterQuery = setValueOn(filterQuery, "${end}", endValue);
Set<String> paramNames = collectParamNames(filterQuery);
if (document != null) {
for (String parameter : paramNames) {
if (document.get(parameter) != null)
- filterQuery = filterQuery.replace(String.format("${%s}", parameter), document.get(parameter));
+ filterQuery = filterQuery.replace(String.format("${%s}", parameter), String.format("\"%s\"", document.get(parameter)));
}
}
@@ -99,10 +109,14 @@ public class SolrQueryBuilder {
return solrQuery;
}
- private String setEndValueOn(String query) {
- if (endValue != null)
- query = query.replace("${end}", endValue);
- return query;
+ private String setValueOn(String query, String placeHolder, String value) {
+ if (isBlank(value)) {
+ value = "*";
+ }
+ else {
+ value = '"' + value + '"';
+ }
+ return query.replace(placeHolder, value);
}
private Set<String> collectParamNames(String filterQuery) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryProperties.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryProperties.java
index 444a15b..f062879 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryProperties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryProperties.java
@@ -19,6 +19,12 @@
package org.apache.ambari.infra.job.archive;
import org.hibernate.validator.constraints.NotBlank;
+import org.springframework.batch.core.JobParameters;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
public class SolrQueryProperties {
@NotBlank
@@ -26,7 +32,7 @@ public class SolrQueryProperties {
@NotBlank
private String queryText;
private String filterQueryText;
- private String[] sort;
+ private String[] sortColumn;
public String getCollection() {
return collection;
@@ -52,18 +58,40 @@ public class SolrQueryProperties {
this.filterQueryText = filterQueryText;
}
- public String[] getSort() {
- return sort;
+ public String[] getSortColumn() {
+ return sortColumn;
}
- public void setSort(String[] sort) {
- this.sort = sort;
+ public void setSortColumn(String[] sortColumn) {
+ this.sortColumn = sortColumn;
}
public SolrQueryBuilder toQueryBuilder() {
return new SolrQueryBuilder().
setQueryText(queryText)
.setFilterQueryText(filterQueryText)
- .addSort(sort);
+ .addSort(sortColumn);
+ }
+
+ public void apply(JobParameters jobParameters) {
+ collection = jobParameters.getString("collection", collection);
+ queryText = jobParameters.getString("queryText", queryText);
+ filterQueryText = jobParameters.getString("filterQueryText", filterQueryText);
+
+ String sortValue;
+ List<String> sortColumns = new ArrayList<>();
+ int i = 0;
+ while ((sortValue = jobParameters.getString(String.format("sortColumn[%d]", i))) != null) {
+ sortColumns.add(sortValue);
+ ++i;
+ }
+
+ if (sortColumns.size() > 0)
+ sortColumn = sortColumns.toArray(new String[sortColumns.size()]);
+ }
+
+ public void validate() {
+ if (isBlank(collection))
+ throw new IllegalArgumentException("The property collection can not be null or empty string!");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
index 8e34ca9..55ba58a 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
@@ -25,7 +25,7 @@ import org.apache.commons.io.IOUtils;
import java.io.*;
-public class TarGzCompressor implements FileAction {
+public class TarGzCompressor extends AbstractFileAction {
@Override
public File perform(File inputFile) {
File tarGzFile = new File(inputFile.getParent(), inputFile.getName() + ".tar.gz");
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
index 7ef70aa..27b36b3 100644
--- a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
+++ b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
@@ -20,13 +20,41 @@ management.security.enabled=false
management.health.solr.enabled=false
infra-manager.server.data.folder=/tmp
-infra-manager.jobs.solr_data_export.zoo_keeper_socket=zookeeper:2181
-infra-manager.jobs.solr_data_export.read_block_size=100
-infra-manager.jobs.solr_data_export.write_block_size=150
-infra-manager.jobs.solr_data_export.file_name_suffix_column=logtime
-infra-manager.jobs.solr_data_export.destination_directory_path=/tmp/ambariInfraManager
-infra-manager.jobs.solr_data_export.query.collection=hadoop_logs
-infra-manager.jobs.solr_data_export.query.query_text=logtime:[* TO "${end}"]
-infra-manager.jobs.solr_data_export.query.filter_query_text=(logtime:"${logtime}" AND id:{"${id}" TO *]) OR logtime:{"${logtime}" TO "${end}"]
-infra-manager.jobs.solr_data_export.query.sort[0]=logtime
-infra-manager.jobs.solr_data_export.query.sort[1]=id
+infra-manager.jobs.solr_data_export.export_service_logs.zoo_keeper_connection_string=zookeeper:2181
+infra-manager.jobs.solr_data_export.export_service_logs.read_block_size=100
+infra-manager.jobs.solr_data_export.export_service_logs.write_block_size=150
+infra-manager.jobs.solr_data_export.export_service_logs.file_name_suffix_column=logtime
+infra-manager.jobs.solr_data_export.export_service_logs.destination_directory_path=/tmp/ambariInfraManager
+infra-manager.jobs.solr_data_export.export_service_logs.query.collection=hadoop_logs
+infra-manager.jobs.solr_data_export.export_service_logs.query.query_text=logtime:[${start} TO ${end}]
+infra-manager.jobs.solr_data_export.export_service_logs.query.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
+infra-manager.jobs.solr_data_export.export_service_logs.query.sort_column[0]=logtime
+infra-manager.jobs.solr_data_export.export_service_logs.query.sort_column[1]=id
+infra-manager.jobs.solr_data_export.export_audit_logs.zoo_keeper_connection_string=zookeeper:2181
+infra-manager.jobs.solr_data_export.export_audit_logs.read_block_size=100
+infra-manager.jobs.solr_data_export.export_audit_logs.write_block_size=150
+# TODO: logtime may not be enough: The same filename can be generated when more than write_block_size count docs has the same logtime value
+infra-manager.jobs.solr_data_export.export_audit_logs.file_name_suffix_column=logtime
+infra-manager.jobs.solr_data_export.export_audit_logs.destination_directory_path=/tmp/ambariInfraManager
+infra-manager.jobs.solr_data_export.export_audit_logs.query.collection=audit_logs
+infra-manager.jobs.solr_data_export.export_audit_logs.query.query_text=logtime:[${start} TO ${end}]
+infra-manager.jobs.solr_data_export.export_audit_logs.query.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
+infra-manager.jobs.solr_data_export.export_audit_logs.query.sort_column[0]=logtime
+infra-manager.jobs.solr_data_export.export_audit_logs.query.sort_column[1]=id
+# TODO: s3_access_key and s3_secret_key to separate file
+infra-manager.jobs.solr_data_export.export_audit_logs.s3_access_key=remote-identity
+infra-manager.jobs.solr_data_export.export_audit_logs.s3_secret_key=remote-credential
+infra-manager.jobs.solr_data_export.export_audit_logs.s3_key_prefix=solr_archive_
+infra-manager.jobs.solr_data_export.export_audit_logs.s3_bucket_name=testbucket
+infra-manager.jobs.solr_data_export.export_audit_logs.s3_endpoint=http://fakes3:4569
+# TODO: configure ranger audit logs
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.zoo_keeper_connection_string=zookeeper:2181
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.read_block_size=100
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.write_block_size=150
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.file_name_suffix_column=logtime
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.destination_directory_path=/tmp/ambariInfraManager
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.collection=hadoop_logs
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.query_text=logtime:[* TO "${end}"]
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.filter_query_text=(logtime:"${logtime}" AND id:{"${id}" TO *]) OR logtime:{"${logtime}" TO "${end}"]
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.sort_column[0]=logtime
+#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.sort_column[1]=id
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java
new file mode 100644
index 0000000..ae93710
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java
@@ -0,0 +1,54 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class DocumentExportPropertiesTest {
+ @Test
+ public void testDeepCopy() throws Exception {
+ DocumentExportProperties documentExportProperties = new DocumentExportProperties();
+ documentExportProperties.setDestinationDirectoryPath("/tmp");
+ documentExportProperties.setFileNameSuffixColumn(".json");
+ documentExportProperties.setReadBlockSize(10);
+ documentExportProperties.setWriteBlockSize(20);
+ documentExportProperties.setZooKeeperConnectionString("localhost:2181");
+ SolrQueryProperties query = new SolrQueryProperties();
+ query.setFilterQueryText("id:1167");
+ query.setQueryText("name:'Joe'");
+ query.setCollection("Users");
+ query.setSortColumn(new String[] {"name"});
+ documentExportProperties.setQuery(query);
+
+ DocumentExportProperties parsed = documentExportProperties.deepCopy();
+
+ assertThat(parsed.getDestinationDirectoryPath(), is(documentExportProperties.getDestinationDirectoryPath()));
+ assertThat(parsed.getFileNameSuffixColumn(), is(documentExportProperties.getFileNameSuffixColumn()));
+ assertThat(parsed.getReadBlockSize(), is(documentExportProperties.getReadBlockSize()));
+ assertThat(parsed.getWriteBlockSize(), is(documentExportProperties.getWriteBlockSize()));
+ assertThat(parsed.getZooKeeperConnectionString(), is(documentExportProperties.getZooKeeperConnectionString()));
+ assertThat(parsed.getQuery().getQueryText(), is(query.getQueryText()));
+ assertThat(parsed.getQuery().getFilterQueryText(), is(query.getFilterQueryText()));
+ assertThat(parsed.getQuery().getCollection(), is(query.getCollection()));
+ assertThat(parsed.getQuery().getSortColumn(), is(query.getSortColumn()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentItemReaderTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentItemReaderTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentItemReaderTest.java
index 942713f..0776c3c 100644
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentItemReaderTest.java
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentItemReaderTest.java
@@ -19,6 +19,8 @@
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.CloseableIterator;
+import org.apache.ambari.infra.job.ObjectSource;
import org.easymock.EasyMockRunner;
import org.easymock.EasyMockSupport;
import org.easymock.Mock;
@@ -45,11 +47,11 @@ public class DocumentItemReaderTest extends EasyMockSupport {
private DocumentItemReader documentItemReader;
@Mock
- private DocumentSource documentSource;
+ private ObjectSource<Document> documentSource;
@Mock
- private DocumentIterator documentIterator;
+ private CloseableIterator<Document> documentIterator;
@Mock
- private DocumentIterator documentIterator2;
+ private CloseableIterator<Document> documentIterator2;
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
index 6411ff1..3af93bc 100644
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
@@ -35,7 +35,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
@@ -49,12 +49,12 @@ public class LocalDocumentItemWriterTest extends EasyMockSupport {
private LocalDocumentItemWriter localDocumentItemWriter;
private File outFile;
@Mock
- private FileAction fileAction;
+ private ItemWriterListener itemWriterListener;
@Before
public void setUp() throws Exception {
outFile = File.createTempFile("LocalDocumentItemWriterTest", "json.tmp");
- localDocumentItemWriter = new LocalDocumentItemWriter(outFile, fileAction);
+ localDocumentItemWriter = new LocalDocumentItemWriter(outFile, itemWriterListener);
}
@After
@@ -65,7 +65,7 @@ public class LocalDocumentItemWriterTest extends EasyMockSupport {
@Test
public void testWrite() throws Exception {
- expect(fileAction.perform(outFile)).andReturn(outFile);
+ itemWriterListener.onCompleted(outFile); expectLastCall();
replayAll();
localDocumentItemWriter.write(DOCUMENT);
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
index 4d5ebf1..e9513dc 100644
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
@@ -48,7 +48,7 @@ public class SolrQueryBuilderTest {
@Test
public void testSetQuery() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
- .setQueryText("logtime:[* TO \"${end}\"]")
+ .setQueryText("logtime:[* TO ${end}]")
.setEndValue("2017-11-27'T'10:12:11.372Z")
.build();
assertThat(solrQuery.getQuery(), is("logtime:[* TO \"2017-11-27'T'10:12:11.372Z\"]"));
@@ -57,7 +57,7 @@ public class SolrQueryBuilderTest {
@Test
public void testSetFilterQuery() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
- .setFilterQueryText("(logtime:\"${logtime}\" AND id:{\"${id}\" TO *]) OR logtime:{\"${logtime}\" TO \"${end}\"]")
+ .setFilterQueryText("(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]")
.setDocument(DOCUMENT)
.setEndValue("2017-11-27'T'10:12:11.372Z")
.build();
@@ -76,7 +76,7 @@ public class SolrQueryBuilderTest {
@Test
public void testSetFilterQueryWhenEndValueIsNull() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
- .setFilterQueryText("logtime:\"${logtime}\" AND id:{\"${id}\" TO *]")
+ .setFilterQueryText("logtime:${logtime} AND id:{${id} TO *]")
.setDocument(DOCUMENT)
.build();
assertThat(solrQuery.getFilterQueries()[0], is("logtime:\"2017-10-02'T'10:00:11.634Z\" AND id:{\"1\" TO *]"));
@@ -110,4 +110,16 @@ public class SolrQueryBuilderTest {
assertThat(solrQuery.getSorts().get(0).getItem(), is("logtime"));
assertThat(solrQuery.getSorts().get(1).getItem(), is("id"));
}
+
+ @Test
+ public void test_start_and_end_values_are_given() throws Exception {
+ SolrQuery solrQuery = new SolrQueryBuilder().setQueryText("id:[${start} TO ${end}]").setInterval("10", "13").build();
+ assertThat(solrQuery.getQuery(), is("id:[\"10\" TO \"13\"]"));
+ }
+
+ @Test
+ public void test_start_and_end_values_are_null() throws Exception {
+ SolrQuery solrQuery = new SolrQueryBuilder().setQueryText("id:[${start} TO ${end}]").build();
+ assertThat(solrQuery.getQuery(), is("id:[* TO *]"));
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryPropertiesTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryPropertiesTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryPropertiesTest.java
new file mode 100644
index 0000000..322775e
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryPropertiesTest.java
@@ -0,0 +1,54 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class SolrQueryPropertiesTest {
+ @Test
+ public void testApplySortColumns() throws Exception {
+ JobParameters jobParameters = new JobParametersBuilder()
+ .addString("sortColumn[0]", "logtime")
+ .addString("sortColumn[1]", "id")
+ .toJobParameters();
+
+ SolrQueryProperties solrQueryProperties = new SolrQueryProperties();
+ solrQueryProperties.setSortColumn(new String[] {"testColumn"});
+ solrQueryProperties.apply(jobParameters);
+ assertThat(solrQueryProperties.getSortColumn().length, is(2));
+ assertThat(solrQueryProperties.getSortColumn()[0], is("logtime"));
+ assertThat(solrQueryProperties.getSortColumn()[1], is("id"));
+ }
+
+ @Test
+ public void testApplyWhenNoSortIsDefined() throws Exception {
+ JobParameters jobParameters = new JobParametersBuilder()
+ .toJobParameters();
+
+ SolrQueryProperties solrQueryProperties = new SolrQueryProperties();
+ solrQueryProperties.setSortColumn(new String[] {"testColumn"});
+ solrQueryProperties.apply(jobParameters);
+ assertThat(solrQueryProperties.getSortColumn().length, is(1));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/36d0271f/ambari-infra/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-infra/pom.xml b/ambari-infra/pom.xml
index e4b6a76..9e7a71b 100644
--- a/ambari-infra/pom.xml
+++ b/ambari-infra/pom.xml
@@ -23,7 +23,6 @@
<version>2.0.0.0-SNAPSHOT</version>
<relativePath>../ambari-project</relativePath>
</parent>
- <groupId>org.apache.ambari</groupId>
<modelVersion>4.0.0</modelVersion>
<artifactId>ambari-infra</artifactId>
<version>2.0.0.0-SNAPSHOT</version>
@@ -44,6 +43,7 @@
<module>ambari-infra-solr-client</module>
<module>ambari-infra-solr-plugin</module>
<module>ambari-infra-manager</module>
+ <module>ambari-infra-manager-it</module>
</modules>
<build>
@@ -161,6 +161,9 @@
<exclude>**/*.json</exclude>
<exclude>**/*.log</exclude>
<exclude>**/*.txt</exclude>
+ <exclude>**/docker/Profile</exclude>
+ <exclude>**/docker/.env</exclude>
+ <exclude>**/*.story</exclude>
</excludes>
</configuration>
<executions>