You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/12/08 23:30:03 UTC
[24/25] ambari git commit: Merge from branch-2.5
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
index 834ba38..da76924 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
@@ -20,8 +20,12 @@ package org.apache.ambari.logsearch.dao;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.common.SolrException;
@@ -33,6 +37,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
import javax.inject.Named;
+import javax.ws.rs.core.Response;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -45,6 +50,8 @@ class SolrCollectionDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
private static final int SETUP_RETRY_SECOND = 30;
+ private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
+ private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
/**
* This will try to get the collections from the Solr. Ping doesn't work if
@@ -189,7 +196,16 @@ class SolrCollectionDao {
}
} else {
LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
- Collection<String> existingShards = getShards(solrClient, solrPropsConfig);
+ Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
+ Collection<String> existingShards = getShards(slices, solrPropsConfig);
+ if (existingShards.size() < shardsList.size()) {
+ try {
+ updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
+ } catch (Throwable t) {
+ returnValue = false;
+ LOG.error(String.format("Exception during updating collection (%s)", t));
+ }
+ }
for (String shard : shardsList) {
if (!existingShards.contains(shard)) {
try {
@@ -216,10 +232,44 @@ class SolrCollectionDao {
return returnValue;
}
- private Collection<String> getShards(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
- Collection<String> list = new HashSet<>();
+ private String getRandomBaseUrl(Collection<Slice> slices) {
+ String coreUrl = null;
+ if (slices != null) {
+ for (Slice slice : slices) {
+ if (!slice.getReplicas().isEmpty()) {
+ Replica replica = slice.getReplicas().iterator().next();
+ coreUrl = replica.getStr("base_url");
+ if (coreUrl != null) {
+ break;
+ }
+ }
+ }
+ }
+ return coreUrl;
+ }
+
+ private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
+ String baseUrl = getRandomBaseUrl(slices);
+ if (baseUrl != null) {
+ CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
+ HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
+ solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
+ HttpResponse response = httpClient.execute(request);
+ if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
+ throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
+ }
+ } else {
+ throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
+ }
+ }
+
+ private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
ZkStateReader reader = solrClient.getZkStateReader();
- Collection<Slice> slices = reader.getClusterState().getSlices(solrPropsConfig.getCollection());
+ return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
+ }
+
+ private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
+ Collection<String> list = new HashSet<>();
for (Slice slice : slices) {
for (Replica replica : slice.getReplicas()) {
LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
@@ -245,7 +295,7 @@ class SolrCollectionDao {
collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
- collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
if (createResponse.getStatus() != 0) {
LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
@@ -256,4 +306,8 @@ class SolrCollectionDao {
return true;
}
}
+
+ private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
+ return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java
new file mode 100644
index 0000000..9fa5c80
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listener/LogSearchSessionListener.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.listener;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpSessionEvent;
+import javax.servlet.http.HttpSessionListener;
+
+public class LogSearchSessionListener implements HttpSessionListener {
+
+ private Logger LOG = LoggerFactory.getLogger(LogSearchSessionListener.class);
+
+ private int numberOfSessions = 0;
+
+ @Override
+ public void sessionCreated(HttpSessionEvent event) {
+ synchronized (this) {
+ numberOfSessions++;
+ }
+ LOG.debug(String.format("New session is created (Id: %s). Number of sessions: %d", event.getSession().getId(), numberOfSessions));
+ }
+
+ @Override
+ public void sessionDestroyed(HttpSessionEvent event) {
+ synchronized (this) {
+ numberOfSessions--;
+ }
+ LOG.debug(String.format("Session destroyed (Id: %s). Number of sessions: %d", event.getSession().getId(), numberOfSessions));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-logsearch/docker/test-config/logsearch/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logsearch/log4j.xml b/ambari-logsearch/docker/test-config/logsearch/log4j.xml
index b80824b..d0e26ed 100644
--- a/ambari-logsearch/docker/test-config/logsearch/log4j.xml
+++ b/ambari-logsearch/docker/test-config/logsearch/log4j.xml
@@ -25,7 +25,7 @@
<param name="maxFileSize" value="10MB" />
<param name="maxBackupIndex" value="10" />
<layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
+ <param name="ConversionPattern" value="%d %-5p [%t] %C{6} (%F:%L) - %m%n" />
</layout>
</appender>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 769d0c1..9bc3be5 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -84,6 +84,10 @@ public abstract class AbstractTimelineMetricsSink {
public int ZK_CONNECT_TRY_COUNT = 10;
public int ZK_SLEEP_BETWEEN_RETRY_TIME = 2000;
public boolean shardExpired = true;
+ private int zookeeperMinBackoffTimeMins = 2;
+ private int zookeeperMaxBackoffTimeMins = 5;
+ private long zookeeperBackoffTimeMillis;
+ private long lastFailedZkRequestTime = 0l;
private SSLSocketFactory sslSocketFactory;
@@ -132,6 +136,7 @@ public abstract class AbstractTimelineMetricsSink {
metricSinkWriteShardStrategy = new MetricSinkWriteShardHostnameHashingStrategy(getHostname());
collectorHAHelper = new MetricCollectorHAHelper(getZookeeperQuorum(),
ZK_CONNECT_TRY_COUNT, ZK_SLEEP_BETWEEN_RETRY_TIME);
+ zookeeperBackoffTimeMillis = getZookeeperBackoffTimeMillis();
isInitializedForHA = true;
}
@@ -217,6 +222,11 @@ public abstract class AbstractTimelineMetricsSink {
collectorHost = findPreferredCollectHost();
}
+ if (collectorHost == null) {
+ LOG.warn("No live collector to send metrics to. Metrics to be sent will be discarded.");
+ return false;
+ }
+
String connectUrl = getCollectorUri(collectorHost);
String jsonData = null;
LOG.debug("EmitMetrics connectUrl = " + connectUrl);
@@ -337,13 +347,27 @@ public abstract class AbstractTimelineMetricsSink {
}
// Reach out to all configured collectors before Zookeeper
- refreshCollectorsFromConfigured();
+ Collection<String> collectorHosts = getConfiguredCollectorHosts();
+ refreshCollectorsFromConfigured(collectorHosts);
// Lookup Zookeeper for live hosts - max 10 seconds wait time
- if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null) {
- //TODO : Bring back Zk fallback after proper curation.
- LOG.info("No live collectors from configuration. Not requesting zookeeper...");
- //allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode());
+ long currentTime = System.currentTimeMillis();
+ if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null
+ && (currentTime - lastFailedZkRequestTime) > zookeeperBackoffTimeMillis) {
+
+ LOG.info("No live collectors from configuration. Requesting zookeeper...");
+ allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode());
+ boolean noNewCollectorFromZk = true;
+ for (String collectorHostFromZk : allKnownLiveCollectors) {
+ if (!collectorHosts.contains(collectorHostFromZk)) {
+ noNewCollectorFromZk = false;
+ break;
+ }
+ }
+ if (noNewCollectorFromZk) {
+ LOG.info("No new collector was found from Zookeeper. Will not request zookeeper for " + zookeeperBackoffTimeMillis + " millis");
+ lastFailedZkRequestTime = System.currentTimeMillis();
+ }
}
if (allKnownLiveCollectors.size() != 0) {
@@ -356,7 +380,7 @@ public abstract class AbstractTimelineMetricsSink {
// OR
// through Expiry (Refresh needed to pick up dead collectors that might have not become alive).
if (shardExpired) {
- refreshCollectorsFromConfigured();
+ refreshCollectorsFromConfigured(getConfiguredCollectorHosts());
}
return metricSinkWriteShardStrategy.findCollectorShard(new ArrayList<>(allKnownLiveCollectors));
}
@@ -376,8 +400,7 @@ public abstract class AbstractTimelineMetricsSink {
return null;
}
- private void refreshCollectorsFromConfigured() {
- Collection<String> collectorHosts = getConfiguredCollectorHosts();
+ private void refreshCollectorsFromConfigured(Collection<String> collectorHosts) {
LOG.debug("Trying to find live collector host from : " + collectorHosts);
if (collectorHosts != null && !collectorHosts.isEmpty()) {
@@ -477,7 +500,7 @@ public abstract class AbstractTimelineMetricsSink {
return sb.toString();
}
/**
- * Parses input Sting of format "['host1', 'host2']" into Collection of hostnames
+ * Parses input Sting of format "host1,host2" into Collection of hostnames
*/
public Collection<String> parseHostsStringIntoCollection(String hostsString) {
Set<String> hosts = new HashSet<>();
@@ -487,10 +510,8 @@ public abstract class AbstractTimelineMetricsSink {
return hosts;
}
- String[] untrimmedHosts = hostsString.split(",");
- for (String host : untrimmedHosts) {
- host = StringUtils.substringBetween(host, "'");
+ for (String host : hostsString.split(",")) {
if (StringUtils.isEmpty(host))
continue;
hosts.add(host.trim());
@@ -499,6 +520,12 @@ public abstract class AbstractTimelineMetricsSink {
return hosts;
}
+
+ private long getZookeeperBackoffTimeMillis() {
+ return (zookeeperMinBackoffTimeMins +
+ rand.nextInt(zookeeperMaxBackoffTimeMins - zookeeperMinBackoffTimeMins + 1)) * 60*1000l;
+ }
+
/**
* Get a pre-formatted URI for the collector
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
index 54c2bdf..c6f6beb 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
@@ -23,7 +23,6 @@ import org.apache.curator.CuratorZookeeperClient;
import org.apache.curator.RetryLoop;
import org.apache.curator.RetryPolicy;
import org.apache.curator.retry.BoundedExponentialBackoffRetry;
-import org.apache.curator.retry.RetryUntilElapsed;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
index 5e016f8..9b0cdbe 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java
@@ -32,21 +32,21 @@ public class AbstractTimelineMetricSinkTest {
AbstractTimelineMetricsSink sink = new TestTimelineMetricsSink();
Collection<String> hosts;
- hosts = sink.parseHostsStringIntoCollection("[]");
+ hosts = sink.parseHostsStringIntoCollection("");
Assert.assertTrue(hosts.isEmpty());
- hosts = sink.parseHostsStringIntoCollection("[u'test1.123.abc.def.local']");
+ hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local");
Assert.assertTrue(hosts.size() == 1);
Assert.assertTrue(hosts.contains("test1.123.abc.def.local"));
- hosts = sink.parseHostsStringIntoCollection("['test1.123.abc.def.local']");
+ hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local ");
Assert.assertTrue(hosts.size() == 1);
Assert.assertTrue(hosts.contains("test1.123.abc.def.local"));
- hosts = sink.parseHostsStringIntoCollection("[u'test1.123.abc.def.local', u'test1.456.abc.def.local']");
+ hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local,test1.456.abc.def.local");
Assert.assertTrue(hosts.size() == 2);
- hosts = sink.parseHostsStringIntoCollection("['test1.123.abc.def.local', 'test1.456.abc.def.local']");
+ hosts = sink.parseHostsStringIntoCollection("test1.123.abc.def.local, test1.456.abc.def.local");
Assert.assertTrue(hosts.size() == 2);
Assert.assertTrue(hosts.contains("test1.123.abc.def.local"));
Assert.assertTrue(hosts.contains("test1.456.abc.def.local"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
index a0bc2e8..a393a96 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
@@ -49,7 +49,6 @@ import static org.powermock.api.easymock.PowerMock.verifyAll;
@PrepareForTest({AbstractTimelineMetricsSink.class, URL.class, HttpURLConnection.class, MetricCollectorHAHelper.class})
public class MetricCollectorHATest {
- @Ignore
@Test
public void findCollectorUsingZKTest() throws Exception {
InputStream is = createNiceMock(InputStream.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
index 36ec074..32fe32e 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java
@@ -130,6 +130,10 @@ public class HandleConnectExceptionTest {
return super.emitMetrics(metrics);
}
+ @Override
+ protected synchronized String findPreferredCollectHost() {
+ return "localhost";
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana b/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana
index 472942e..8a8ac8f 100644
--- a/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana
+++ b/ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana
@@ -104,7 +104,7 @@ case "$1" in
# Prepare environment
# mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$LOG_DIR" "$DATA_DIR"
- # touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE"
+ touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE"
# if [ -n "$MAX_OPEN_FILES" ]; then
# ulimit -n $MAX_OPEN_FILES
@@ -116,10 +116,10 @@ case "$1" in
return=$?
if [ $return -eq 0 ]
then
- sleep 1
+ sleep 5
# check if pid file has been written two
if ! [[ -s $PID_FILE ]]; then
- echo "Start FAILED" >> $LOG_FILE
+ echo "Start FAILED because daemon did not write pid in pid_file" >> $LOG_FILE
exit 1
fi
i=0
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
index 802e684..b2ca52e 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
@@ -194,13 +194,13 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
}
/**
- * Parses input Stings array of format "['host1'", '"host2']" into Collection of hostnames
+ * Parses input Stings array of format "host1,host2" into Collection of hostnames
*/
protected Collection<String> parseHostsStringArrayIntoCollection(String[] hostStrings) {
Collection<String> result = new HashSet<>();
if (hostStrings == null) return result;
for (String s : hostStrings) {
- result.addAll(parseHostsStringIntoCollection(s));
+ result.add(s.trim());
}
return result;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
index da00155..0cafd9a 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
@@ -100,7 +100,7 @@ public class HadoopTimelineMetricsSinkTest {
expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes();
expect(conf.getParent()).andReturn(null).anyTimes();
expect(conf.getPrefix()).andReturn("service").anyTimes();
- expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes();
+ expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"localhost"," localhost2"}).anyTimes();
expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes();
expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes();
expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes();
@@ -172,7 +172,7 @@ public class HadoopTimelineMetricsSinkTest {
expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes();
expect(conf.getParent()).andReturn(null).anyTimes();
expect(conf.getPrefix()).andReturn("service").anyTimes();
- expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes();
+ expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"localhost", "localhost2"}).anyTimes();
expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes();
expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes();
expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes();
@@ -302,7 +302,7 @@ public class HadoopTimelineMetricsSinkTest {
expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes();
expect(conf.getParent()).andReturn(null).anyTimes();
expect(conf.getPrefix()).andReturn("service").anyTimes();
- expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes();
+ expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"localhost", "localhost2"}).anyTimes();
expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes();
expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes();
expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
index 9f95c26..7010187 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
@@ -102,7 +102,7 @@ config_content = """
[default]
debug_level = INFO
hostname = localhost
-metrics_servers = ['localhost','host1','host2']
+metrics_servers = localhost
enable_time_threshold = false
enable_value_threshold = false
@@ -217,16 +217,13 @@ class Configuration:
def get_metrics_collector_hosts(self):
hosts = self.get("default", "metrics_servers", "localhost")
- if hosts is not "localhost":
- return ast.literal_eval(hosts)
- else:
- return hosts
+ return hosts.split(",")
def get_failover_strategy(self):
return self.get("collector", "failover_strategy", ROUND_ROBIN_FAILOVER_STRATEGY)
def get_failover_strategy_blacklisted_interval_seconds(self):
- return self.get("collector", "failover_strategy_blacklisted_interval_seconds", 600)
+ return self.get("collector", "failover_strategy_blacklisted_interval_seconds", 300)
def get_hostname_script(self):
if self.hostname_script:
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py
index de97292..4056ae3 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py
@@ -83,7 +83,7 @@ class TestEmitter(TestCase):
emitter.RETRY_SLEEP_INTERVAL = .001
emitter.submit_metrics()
- self.assertEqual(request_mock.call_count, 9)
+ self.assertEqual(request_mock.call_count, 3)
self.assertUrlData(request_mock)
def assertUrlData(self, request_mock):
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index 75ce845..844ee41 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -107,16 +107,14 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
// Initialize policies before TTL update
hBaseAccessor.initPoliciesAndTTL();
// Start HA service
- if (configuration.isDistributedOperationModeEnabled()) {
- // Start the controller
- haController = new MetricCollectorHAController(configuration);
- try {
- haController.initializeHAController();
- } catch (Exception e) {
- LOG.error(e);
- throw new MetricsSystemInitializationException("Unable to " +
- "initialize HA controller", e);
- }
+ // Start the controller
+ haController = new MetricCollectorHAController(configuration);
+ try {
+ haController.initializeHAController();
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new MetricsSystemInitializationException("Unable to " +
+ "initialize HA controller", e);
}
String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, "");
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 9858e4c..27caf88 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -320,6 +320,20 @@ public class TimelineMetricConfiguration {
return hbaseConf.getTrimmed("hbase.zookeeper.quorum");
}
+ public String getClusterZKClientPort() throws MalformedURLException, URISyntaxException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return metricsConf.getTrimmed("cluster.zookeeper.property.clientPort", "2181");
+ }
+
+ public String getClusterZKQuorum() throws MalformedURLException, URISyntaxException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return metricsConf.getTrimmed("cluster.zookeeper.quorum");
+ }
+
public String getInstanceHostnameFromEnv() throws UnknownHostException {
String amsInstanceName = System.getProperty("AMS_INSTANCE_NAME");
if (amsInstanceName == null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
index de63d4e..62f8137 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
@@ -317,9 +317,9 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg
protected void downsample(Connection conn, Long startTime, Long endTime) {
- LOG.info("Checking for downsampling requests.");
+ LOG.debug("Checking for downsampling requests.");
if (CollectionUtils.isEmpty(configuredDownSamplers)) {
- LOG.info("No downsamplers configured");
+ LOG.debug("No downsamplers configured");
return;
}
@@ -424,7 +424,7 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg
PreparedStatement stmt = null;
ResultSet rs = null;
- LOG.info("Downsampling query : " + condition.getStatement());
+ LOG.debug("Downsampling query : " + condition.getStatement());
try {
stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
index edce367..26702b6 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
@@ -81,8 +81,8 @@ public class MetricCollectorHAController {
}
try {
- String zkClientPort = configuration.getZKClientPort();
- String zkQuorum = configuration.getZKQuorum();
+ String zkClientPort = configuration.getClusterZKClientPort();
+ String zkQuorum = configuration.getClusterZKQuorum();
if (StringUtils.isEmpty(zkClientPort) || StringUtils.isEmpty(zkQuorum)) {
throw new Exception("Unable to parse zookeeper quorum. clientPort = "
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
index 462213d..3688630 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.HBaseTimelineMetricStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultPhoenixDataSource;
import org.apache.zookeeper.ClientCnxn;
import org.easymock.EasyMock;
@@ -71,7 +73,7 @@ import static org.powermock.api.support.membermodification.MemberMatcher.method;
import static org.powermock.api.support.membermodification.MemberModifier.suppress;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ PhoenixHBaseAccessor.class, UserGroupInformation.class,
+@PrepareForTest({ PhoenixHBaseAccessor.class, HBaseTimelineMetricStore.class, UserGroupInformation.class,
ClientCnxn.class, DefaultPhoenixDataSource.class, ConnectionFactory.class,
TimelineMetricConfiguration.class, ApplicationHistoryServer.class })
@PowerMockIgnore( {"javax.management.*"})
@@ -179,6 +181,8 @@ public class TestApplicationHistoryServer {
expect(metricConfiguration.getTimelineMetricsServiceHandlerThreadCount()).andReturn(20).anyTimes();
expect(metricConfiguration.getWebappAddress()).andReturn("localhost:9990").anyTimes();
expect(metricConfiguration.getTimelineServiceRpcAddress()).andReturn("localhost:10299").anyTimes();
+ expect(metricConfiguration.getClusterZKQuorum()).andReturn("localhost").anyTimes();
+ expect(metricConfiguration.getClusterZKClientPort()).andReturn("2181").anyTimes();
Connection connection = createNiceMock(Connection.class);
Statement stmt = createNiceMock(Statement.class);
@@ -197,6 +201,14 @@ public class TestApplicationHistoryServer {
connection.close();
expectLastCall();
+ MetricCollectorHAController haControllerMock = PowerMock.createMock(MetricCollectorHAController.class);
+ expectNew(MetricCollectorHAController.class, metricConfiguration)
+ .andReturn(haControllerMock);
+
+ haControllerMock.initializeHAController();
+ expectLastCall().once();
+ expect(haControllerMock.isInitialized()).andReturn(false).anyTimes();
+
org.apache.hadoop.hbase.client.Connection conn = createNiceMock(org.apache.hadoop.hbase.client.Connection.class);
mockStatic(ConnectionFactory.class);
expect(ConnectionFactory.createConnection((Configuration) anyObject())).andReturn(conn);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
index 1e4bac0..875f013 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
@@ -51,8 +51,8 @@ public class MetricCollectorHAControllerTest extends AbstractMiniHBaseClusterTes
String port = zkUrl.split(":")[3];
String quorum = zkUrl.split(":")[2];
- expect(configuration.getZKClientPort()).andReturn(port);
- expect(configuration.getZKQuorum()).andReturn(quorum);
+ expect(configuration.getClusterZKClientPort()).andReturn(port);
+ expect(configuration.getClusterZKQuorum()).andReturn(quorum);
replay(configuration);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-project/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml
index a38c6a2..52c9992 100644
--- a/ambari-project/pom.xml
+++ b/ambari-project/pom.xml
@@ -30,6 +30,8 @@
<ambari.dir>${project.parent.basedir}</ambari.dir>
<powermock.version>1.6.3</powermock.version>
<jetty.version>8.1.19.v20160209</jetty.version>
+ <checkstyle.version>6.19</checkstyle.version> <!-- last version that does not require Java 8 -->
+ <checkstyle.skip>false</checkstyle.skip>
</properties>
<profiles>
<profile>
@@ -477,6 +479,11 @@
<artifactId>jline</artifactId>
<version>2.11</version>
</dependency>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<build>
@@ -486,6 +493,40 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>2.17</version>
+ <executions>
+ <execution>
+ <id>checkstyle</id>
+ <phase>test</phase>
+ <configuration>
+ <configLocation>${project.basedir}/checkstyle.xml</configLocation>
+ <encoding>UTF-8</encoding>
+ <consoleOutput>true</consoleOutput>
+ <failsOnError>true</failsOnError>
+ <linkXRef>false</linkXRef>
+ <skip>${checkstyle.skip}</skip>
+ </configuration>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>utility</groupId>
+ <artifactId>utility</artifactId>
+ <version>1.0.0.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/checkstyle.xml
----------------------------------------------------------------------
diff --git a/ambari-server/checkstyle.xml b/ambari-server/checkstyle.xml
new file mode 100644
index 0000000..81f6380
--- /dev/null
+++ b/ambari-server/checkstyle.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<!-- Licensed under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may obtain
+ a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+ required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ See accompanying LICENSE file. -->
+<!DOCTYPE module PUBLIC
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+ <module name="TreeWalker">
+ <module name="AvoidTransactionalOnPrivateMethodsCheck"/>
+ </module>
+</module>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index d1277c3..08757a2 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -686,6 +686,10 @@
<artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
</plugins>
<resources>
<resource>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java
index 2980f38..01195bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentRequests.java
@@ -45,9 +45,9 @@ public class AgentRequests {
public void setExecutionDetailsRequest(String host, String component, String requestExecutionCmd) {
if (StringUtils.isNotBlank(requestExecutionCmd)) {
- LOG.debug("Setting need for exec command to " + requestExecutionCmd + " for " + component);
Map<String, Boolean> perHostRequiresExecCmdDetails = getPerHostRequiresExecCmdDetails(host);
if (Boolean.TRUE.toString().toUpperCase().equals(requestExecutionCmd.toUpperCase())) {
+ LOG.info("Setting need for exec command to " + requestExecutionCmd + " for " + component);
perHostRequiresExecCmdDetails.put(component, Boolean.TRUE);
} else {
perHostRequiresExecCmdDetails.put(component, Boolean.FALSE);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 29737ee..ef1ee4f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -343,6 +343,7 @@ public class ExecutionCommand extends AgentCommand {
String SCRIPT_TYPE = "script_type";
String SERVICE_PACKAGE_FOLDER = "service_package_folder";
String HOOKS_FOLDER = "hooks_folder";
+ String CUSTOM_FOLDER = "custom_folder";
String STACK_NAME = "stack_name";
String SERVICE_TYPE = "service_type";
String STACK_VERSION = "stack_version";
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
index 7f24bf4..45497b5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
@@ -236,10 +236,10 @@ public class CheckDescription {
public static CheckDescription HARDCODED_STACK_VERSION_PROPERTIES_CHECK = new CheckDescription("HARDCODED_STACK_VERSION_PROPERTIES_CHECK",
PrereqCheckType.CLUSTER,
- "Found hardcoded hdp stack version in property value.",
+ "Found hardcoded stack version in property value.",
new ImmutableMap.Builder<String, String>()
.put(AbstractCheckDescriptor.DEFAULT,
- "Some properties seem to contain hardcoded hdp version string \"%s\"." +
+ "Some properties seem to contain hardcoded stack version string \"%s\"." +
" That is a potential problem when doing stack update.").build());
public static CheckDescription VERSION_MISMATCH = new CheckDescription("VERSION_MISMATCH",
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 2a78905..c3cd82e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -25,6 +25,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
@@ -54,7 +55,6 @@ import java.util.EnumMap;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
@@ -79,10 +79,10 @@ import org.apache.ambari.server.ServiceComponentNotFoundException;
import org.apache.ambari.server.ServiceNotFoundException;
import org.apache.ambari.server.StackAccessException;
import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.actionmanager.CommandExecutionType;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.actionmanager.Stage;
-import org.apache.ambari.server.actionmanager.CommandExecutionType;
import org.apache.ambari.server.actionmanager.StageFactory;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
@@ -894,17 +894,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
@Override
public Config createConfig(Cluster cluster, String type, Map<String, String> properties,
String versionTag, Map<String, Map<String, String>> propertiesAttributes) {
- Config config = configFactory.createNew(cluster, type,
- properties, propertiesAttributes);
- if (!StringUtils.isEmpty(versionTag)) {
- config.setTag(versionTag);
- }
-
- config.persist();
+ Config config = configFactory.createNew(cluster, type, versionTag, properties,
+ propertiesAttributes);
cluster.addConfig(config);
-
return config;
}
@@ -1253,7 +1247,12 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
throw new HostNotFoundException(cluster.getClusterName(), sch.getHostName());
}
- r.setMaintenanceState(maintenanceStateHelper.getEffectiveState(sch, host).name());
+ MaintenanceState effectiveMaintenanceState = maintenanceStateHelper.getEffectiveState(sch, host);
+ if(filterByMaintenanceState(request, effectiveMaintenanceState)) {
+ continue;
+ }
+ r.setMaintenanceState(effectiveMaintenanceState.name());
+
response.add(r);
} catch (ServiceComponentHostNotFoundException e) {
if (request.getServiceName() == null || request.getComponentName() == null) {
@@ -1304,7 +1303,12 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
throw new HostNotFoundException(cluster.getClusterName(), sch.getHostName());
}
- r.setMaintenanceState(maintenanceStateHelper.getEffectiveState(sch, host).name());
+ MaintenanceState effectiveMaintenanceState = maintenanceStateHelper.getEffectiveState(sch, host);
+ if(filterByMaintenanceState(request, effectiveMaintenanceState)) {
+ continue;
+ }
+ r.setMaintenanceState(effectiveMaintenanceState.name());
+
response.add(r);
}
}
@@ -1313,6 +1317,24 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
return response;
}
+ private boolean filterByMaintenanceState(ServiceComponentHostRequest request, MaintenanceState effectiveMaintenanceState) {
+ if (request.getMaintenanceState() != null) {
+ MaintenanceState desiredMaintenanceState = MaintenanceState.valueOf(request.getMaintenanceState());
+ if (desiredMaintenanceState.equals(MaintenanceState.ON)) {
+ /*
+ * if we want components with ON state it can be one of IMPLIED_FROM_SERVICE,
+ * IMPLIED_FROM_SERVICE_AND_HOST, IMPLIED_FROM_HOST, ON, ro simply - not OFF
+ */
+ if (effectiveMaintenanceState.equals(MaintenanceState.OFF)) {
+ return true;
+ }
+ } else if (!desiredMaintenanceState.equals(effectiveMaintenanceState)){
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
public MaintenanceState getEffectiveMaintenanceState(ServiceComponentHost sch)
throws AmbariException {
@@ -2217,6 +2239,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
serviceInfo.getServicePackageFolder());
commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
+ String customCacheDirectory = componentInfo.getCustomFolder();
+ if (customCacheDirectory != null) {
+ File customCache = new File(configs.getResourceDirPath(), customCacheDirectory);
+ if (customCache.exists() && customCache.isDirectory()) {
+ commandParams.put(CUSTOM_FOLDER, customCacheDirectory);
+ }
+ }
+
String clusterName = cluster.getClusterName();
if (customCommandExecutionHelper.isTopologyRefreshRequired(roleCommand.name(), clusterName, serviceName)) {
commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True");
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
index 8e5674e..4010528 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
@@ -374,7 +374,7 @@ public class AlertTargetResourceProvider extends
*/
@Transactional
@SuppressWarnings("unchecked")
- private void updateAlertTargets(long alertTargetId,
+ void updateAlertTargets(long alertTargetId,
Map<String, Object> requestMap)
throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index ec846f8..b26e056 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2526,9 +2526,10 @@ public class BlueprintConfigurationProcessor {
}
boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS");
+ boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook"));
// Append atlas hook if not already present.
- if (isAtlasInCluster) {
+ if (isAtlasInCluster || isAtlasHiveHookEnabled) {
if (!hiveHooksClean.contains(atlasHookClass)) {
hiveHooksClean.add(atlasHookClass);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
index 3a86aef..3c415df 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
@@ -53,6 +53,17 @@ public class CalculatedStatus {
*/
private final double percent;
+ /**
+ * A status which represents a COMPLETED state at 0%
+ */
+ public static final CalculatedStatus COMPLETED = new CalculatedStatus(HostRoleStatus.COMPLETED,
+ HostRoleStatus.COMPLETED, 100.0);
+
+ /**
+ * A status which represents a PENDING state at 0%
+ */
+ public static final CalculatedStatus PENDING = new CalculatedStatus(HostRoleStatus.PENDING,
+ HostRoleStatus.PENDING, 0.0);
// ----- Constructors ------------------------------------------------------
@@ -79,12 +90,6 @@ public class CalculatedStatus {
this.percent = percent;
}
- /**
- * Static factory method to get Status that represents a Completed state
- */
- public static CalculatedStatus getCompletedStatus() {
- return new CalculatedStatus(HostRoleStatus.COMPLETED, HostRoleStatus.COMPLETED, 100.0);
- }
// ----- CalculatedStatus --------------------------------------------------
@@ -291,14 +296,25 @@ public class CalculatedStatus {
}
/**
- * Calculates the overall status of an upgrade.
- * @param stageDto the map of stage-to-summary value objects
- * @param stageIds the stage ids to consider from the value objects
+ * Calculates the overall status of an upgrade. If there are no tasks, then a
+ * status of {@link HostRoleStatus#COMPLETED} is returned.
+ *
+ * @param stageDto
+ * the map of stage-to-summary value objects
+ * @param stageIds
+ * the stage ids to consider from the value objects
* @return the calculated status
*/
public static CalculatedStatus statusFromStageSummary(Map<Long, HostRoleCommandStatusSummaryDTO> stageDto,
Set<Long> stageIds) {
+ // if either are empty, then we have no tasks and therefore no status - we
+ // should return COMPLETED. This can happen if someone removes all tasks but
+ // leaves the stages and request
+ if (stageDto.isEmpty() || stageIds.isEmpty()) {
+ return COMPLETED;
+ }
+
Collection<HostRoleStatus> stageStatuses = new HashSet<>();
Collection<HostRoleStatus> stageDisplayStatuses = new HashSet<>();
Collection<HostRoleStatus> taskStatuses = new ArrayList<>();
@@ -378,19 +394,28 @@ public class CalculatedStatus {
*/
public static HostRoleStatus calculateSummaryStatusOfStage(Map<HostRoleStatus, Integer> counters,
int total, boolean skippable) {
+
+ // when there are 0 tasks, return COMPLETED
+ if (total == 0) {
+ return HostRoleStatus.COMPLETED;
+ }
+
if (counters.get(HostRoleStatus.PENDING) == total) {
return HostRoleStatus.PENDING;
}
+
// By definition, any tasks in a future stage must be held in a PENDING status.
if (counters.get(HostRoleStatus.HOLDING) > 0 || counters.get(HostRoleStatus.HOLDING_FAILED) > 0 || counters.get(HostRoleStatus.HOLDING_TIMEDOUT) > 0) {
return counters.get(HostRoleStatus.HOLDING) > 0 ? HostRoleStatus.HOLDING :
counters.get(HostRoleStatus.HOLDING_FAILED) > 0 ? HostRoleStatus.HOLDING_FAILED :
HostRoleStatus.HOLDING_TIMEDOUT;
}
+
// Because tasks are not skippable, guaranteed to be FAILED
if (counters.get(HostRoleStatus.FAILED) > 0 && !skippable) {
return HostRoleStatus.FAILED;
}
+
// Because tasks are not skippable, guaranteed to be TIMEDOUT
if (counters.get(HostRoleStatus.TIMEDOUT) > 0 && !skippable) {
return HostRoleStatus.TIMEDOUT;
@@ -401,9 +426,11 @@ public class CalculatedStatus {
if (counters.get(HostRoleStatus.ABORTED) > 0 && numActiveTasks == 0) {
return HostRoleStatus.ABORTED;
}
+
if (counters.get(HostRoleStatus.COMPLETED) == total) {
return HostRoleStatus.COMPLETED;
}
+
return HostRoleStatus.IN_PROGRESS;
}
@@ -415,7 +442,8 @@ public class CalculatedStatus {
*
* @return summary request status based on statuses of tasks in different states.
*/
- private static HostRoleStatus calculateSummaryStatusOfUpgrade(Map<HostRoleStatus, Integer> counters, int total) {
+ protected static HostRoleStatus calculateSummaryStatusOfUpgrade(
+ Map<HostRoleStatus, Integer> counters, int total) {
return calculateSummaryStatusOfStage(counters, total, false);
}
@@ -428,8 +456,8 @@ public class CalculatedStatus {
*
* @return summary request status based on statuses of tasks in different states.
*/
- private static HostRoleStatus calculateSummaryDisplayStatus(Map<HostRoleStatus, Integer> counters,
- int total, boolean skippable) {
+ protected static HostRoleStatus calculateSummaryDisplayStatus(
+ Map<HostRoleStatus, Integer> counters, int total, boolean skippable) {
return counters.get(HostRoleStatus.SKIPPED_FAILED) > 0 ? HostRoleStatus.SKIPPED_FAILED :
counters.get(HostRoleStatus.FAILED) > 0 ? HostRoleStatus.FAILED:
calculateSummaryStatusOfStage(counters, total, skippable);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index 42aae84..d417ec2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -442,7 +442,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- private void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
+ void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
String desiredRepoVersion, RepositoryVersionState repoState)
throws AmbariException, SystemException {
final String clusterName = cluster.getClusterName();
@@ -482,7 +482,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- private RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
+ RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
List<Host> hosts, RepositoryVersionEntity repoVersionEnt, Map<String, Object> propertyMap)
throws AmbariException, SystemException {
final AmbariManagementController managementController = getManagementController();
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java
index 96bb8f9..2373068 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java
@@ -17,7 +17,16 @@
*/
package org.apache.ambari.server.controller.internal;
-import com.google.inject.Inject;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
import org.apache.ambari.server.ConfigGroupNotFoundException;
@@ -48,7 +57,7 @@ import org.apache.ambari.server.security.authorization.RoleAuthorization;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.ConfigImpl;
+import org.apache.ambari.server.state.ConfigFactory;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.state.configgroup.ConfigGroupFactory;
@@ -56,15 +65,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.inject.Inject;
@StaticallyInject
public class ConfigGroupResourceProvider extends
@@ -102,6 +103,12 @@ public class ConfigGroupResourceProvider extends
private static HostDAO hostDAO;
/**
+ * Used for creating {@link Config} instances to return in the REST response.
+ */
+ @Inject
+ private static ConfigFactory configFactory;
+
+ /**
* Create a new resource provider for the given management controller.
*
* @param propertyIds the property ids
@@ -568,22 +575,19 @@ public class ConfigGroupResourceProvider extends
}
}
+ configLogger.info("User {} is creating new configuration group {} for tag {} in cluster {}",
+ getManagementController().getAuthName(), request.getGroupName(), request.getTag(),
+ cluster.getClusterName());
+
ConfigGroup configGroup = configGroupFactory.createNew(cluster,
request.getGroupName(),
request.getTag(), request.getDescription(),
request.getConfigs(), hosts);
- verifyConfigs(configGroup.getConfigurations(), cluster.getClusterName());
configGroup.setServiceName(serviceName);
- // Persist before add, since id is auto-generated
- configLogger.info("Persisting new Config group"
- + ", clusterName = " + cluster.getClusterName()
- + ", name = " + configGroup.getName()
- + ", tag = " + configGroup.getTag()
- + ", user = " + getManagementController().getAuthName());
+ verifyConfigs(configGroup.getConfigurations(), cluster.getClusterName());
- configGroup.persist();
cluster.addConfigGroup(configGroup);
if (serviceName != null) {
cluster.createServiceConfigVersion(serviceName, getManagementController().getAuthName(),
@@ -634,6 +638,11 @@ public class ConfigGroupResourceProvider extends
+ ", clusterName = " + request.getClusterName()
+ ", groupId = " + request.getId());
}
+
+ configLogger.info("User {} is updating configuration group {} for tag {} in cluster {}",
+ getManagementController().getAuthName(), request.getGroupName(), request.getTag(),
+ cluster.getClusterName());
+
String serviceName = configGroup.getServiceName();
String requestServiceName = cluster.getServiceForConfigTypes(request.getConfigs().keySet());
if (StringUtils.isEmpty(serviceName) && StringUtils.isEmpty(requestServiceName)) {
@@ -682,13 +691,6 @@ public class ConfigGroupResourceProvider extends
configGroup.setDescription(request.getDescription());
configGroup.setTag(request.getTag());
- configLogger.info("Persisting updated Config group"
- + ", clusterName = " + configGroup.getClusterName()
- + ", id = " + configGroup.getId()
- + ", tag = " + configGroup.getTag()
- + ", user = " + getManagementController().getAuthName());
-
- configGroup.persist();
if (serviceName != null) {
cluster.createServiceConfigVersion(serviceName, getManagementController().getAuthName(),
request.getServiceConfigVersionNote(), configGroup);
@@ -781,11 +783,7 @@ public class ConfigGroupResourceProvider extends
}
}
- Config config = new ConfigImpl(type);
- config.setTag(tag);
- config.setProperties(configProperties);
- config.setPropertiesAttributes(configAttributes);
-
+ Config config = configFactory.createReadOnly(type, tag, configProperties, configAttributes);
configurations.put(config.getType(), config);
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
index 8c1bc57..0690ee7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
@@ -69,10 +69,10 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.topology.LogicalRequest;
import org.apache.ambari.server.topology.TopologyManager;
+import org.apache.commons.lang.StringUtils;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
-import org.apache.commons.lang.StringUtils;
/**
* Resource provider for request resources.
@@ -566,13 +566,18 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
ServiceComponentTuple that = (ServiceComponentTuple) o;
- if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null)
+ if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) {
return false;
+ }
return !(componentName != null ? !componentName.equals(that.componentName) : that.componentName != null);
}
@@ -739,15 +744,20 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider
// get summaries from TopologyManager for logical requests
summary.putAll(topologyManager.getStageSummaries(entity.getRequestId()));
+ // summary might be empty due to delete host have cleared all
+ // HostRoleCommands or due to hosts haven't registered yet with the cluster
+ // when the cluster is provisioned with a Blueprint
+ final CalculatedStatus status;
LogicalRequest logicalRequest = topologyManager.getRequest(entity.getRequestId());
-
- CalculatedStatus status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
- if (summary.isEmpty() && logicalRequest == null) {
-
- // summary might be empty due to delete host have cleared all HostRoleCommands
- // or due to hosts haven't registered yet with the cluster when the cluster is provisioned
- // with a Blueprint
- status = CalculatedStatus.getCompletedStatus();
+ if (summary.isEmpty() && null != logicalRequest) {
+ // in this case, it appears that there are no tasks but this is a logical
+ // topology request, so it's a matter of hosts simply not registering yet
+ // for tasks to be created
+ status = CalculatedStatus.PENDING;
+ } else {
+ // there are either tasks or this is not a logical request, so do normal
+ // status calculations
+ status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
}
setResourceProperty(resource, REQUEST_STATUS_PROPERTY_ID, status.getStatus().toString(), requestedPropertyIds);
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
index 59dd9d9..a778882 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
@@ -339,7 +339,7 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
CalculatedStatus status;
if (summary.isEmpty()) {
// Delete host might have cleared all HostRoleCommands
- status = CalculatedStatus.getCompletedStatus();
+ status = CalculatedStatus.COMPLETED;
} else {
status = CalculatedStatus.statusFromStageSummary(summary, Collections.singleton(entity.getStageId()));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index fd38d17..a45ccff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -956,7 +956,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
@Transactional
- private UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
+ UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
RequestStageContainer request,
UpgradeEntity upgradeEntity) throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
index 1c135b2..5c0bdb1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java
@@ -23,9 +23,9 @@ import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractService;
import com.google.inject.Inject;
import org.apache.ambari.server.AmbariService;
-import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
+import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,9 +64,6 @@ public class LogSearchDataRetrievalService extends AbstractService {
private static Logger LOG = LoggerFactory.getLogger(LogSearchDataRetrievalService.class);
@Inject
- private Configuration configuration;
-
- @Inject
private LoggingRequestHelperFactory loggingRequestHelperFactory;
/**
@@ -288,7 +285,7 @@ public class LogSearchDataRetrievalService extends AbstractService {
helper.sendGetLogFileNamesRequest(component, host);
// update the cache if result is available
- if (logFileNamesResult != null) {
+ if (CollectionUtils.isNotEmpty(logFileNamesResult)) {
LOG.debug("LogSearchFileNameRequestRunnable: request was successful, updating cache");
final String key = generateKey(component, host);
// update cache with returned result
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java
new file mode 100644
index 0000000..a779068
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingCookieStore.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.server.controller.logging;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Package protected singleton for storing Cookie key value pairs for Logging Service.
+ * This has chosen instead of using CookieManager to avoid using system wide Cookie handling
+ */
+class LoggingCookieStore {
+ public static final LoggingCookieStore INSTANCE = new LoggingCookieStore();
+
+ private final Map<String, String> cookiesMap = new HashMap<>();
+
+ private LoggingCookieStore() {
+ }
+
+ public Map<String, String> getCookiesMap() {
+ return cookiesMap;
+ }
+
+ public void addCookie(String cookieName, String cookieValue) {
+ cookiesMap.put(cookieName, cookieValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4278c4a4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
index eab0c04..358c1b7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
@@ -26,6 +26,7 @@ import org.apache.ambari.server.security.encryption.CredentialStoreService;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Config;
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.StringUtils;
import org.apache.http.client.utils.URIBuilder;
import org.apache.log4j.Logger;
import org.codehaus.jackson.map.AnnotationIntrospector;
@@ -39,11 +40,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
+import java.net.HttpCookie;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -76,6 +80,10 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
private static final String PAGE_SIZE_QUERY_PARAMETER_NAME = "pageSize";
+ private static final String COOKIE_HEADER = "Cookie";
+
+ private static final String SET_COOKIES_HEADER = "Set-Cookie";
+
private static final int DEFAULT_LOGSEARCH_CONNECT_TIMEOUT_IN_MILLISECONDS = 5000;
private static final int DEFAULT_LOGSEARCH_READ_TIMEOUT_IN_MILLISECONDS = 5000;
@@ -109,20 +117,20 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
// use the Apache builder to create the correct URI
URI logSearchURI = createLogSearchQueryURI("http", queryParameters);
LOG.debug("Attempting to connect to LogSearch server at " + logSearchURI);
-
- HttpURLConnection httpURLConnection = (HttpURLConnection)logSearchURI.toURL().openConnection();
+ HttpURLConnection httpURLConnection = (HttpURLConnection) logSearchURI.toURL().openConnection();
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setConnectTimeout(DEFAULT_LOGSEARCH_CONNECT_TIMEOUT_IN_MILLISECONDS);
httpURLConnection.setReadTimeout(DEFAULT_LOGSEARCH_READ_TIMEOUT_IN_MILLISECONDS);
+ addCookiesFromCookieStore(httpURLConnection);
setupCredentials(httpURLConnection);
StringBuffer buffer = networkConnection.readQueryResponseFromServer(httpURLConnection);
+ addCookiesToCookieStoreFromResponse(httpURLConnection);
// setup a reader for the JSON response
- StringReader stringReader =
- new StringReader(buffer.toString());
+ StringReader stringReader = new StringReader(buffer.toString());
ObjectReader logQueryResponseReader =
createObjectReader(LogQueryResponse.class);
@@ -137,6 +145,27 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
return null;
}
+ private void addCookiesFromCookieStore(HttpURLConnection httpURLConnection) {
+ if (LoggingCookieStore.INSTANCE.getCookiesMap().size() > 0) {
+ List<String> cookiesStrList = new ArrayList<>();
+ for (Map.Entry<String, String> entry : LoggingCookieStore.INSTANCE.getCookiesMap().entrySet()) {
+ cookiesStrList.add(String.format("%s=%s", entry.getKey(), entry.getValue()));
+ }
+ httpURLConnection.setRequestProperty(COOKIE_HEADER, StringUtils.join(cookiesStrList, "; "));
+ }
+ }
+
+ private void addCookiesToCookieStoreFromResponse(HttpURLConnection httpURLConnection) {
+ Map<String, List<String>> headerFields = httpURLConnection.getHeaderFields();
+ List<String> cookiesHeader = headerFields.get(SET_COOKIES_HEADER);
+ if (cookiesHeader != null) {
+ for (String cookie : cookiesHeader) {
+ HttpCookie cookie1 = HttpCookie.parse(cookie).get(0);
+ LoggingCookieStore.INSTANCE.addCookie(cookie1.getName(), cookie1.getValue());
+ }
+ }
+ }
+
private void setupCredentials(HttpURLConnection httpURLConnection) {
final String logSearchAdminUser =
@@ -224,10 +253,14 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
HttpURLConnection httpURLConnection = (HttpURLConnection) logLevelQueryURI.toURL().openConnection();
httpURLConnection.setRequestMethod("GET");
+ addCookiesFromCookieStore(httpURLConnection);
+
setupCredentials(httpURLConnection);
StringBuffer buffer = networkConnection.readQueryResponseFromServer(httpURLConnection);
+ addCookiesToCookieStoreFromResponse(httpURLConnection);
+
// setup a reader for the JSON response
StringReader stringReader =
new StringReader(buffer.toString());
@@ -374,6 +407,7 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream));
LOG.debug("Response code from LogSearch Service is = " + httpURLConnection.getResponseCode());
+
String line = reader.readLine();
StringBuffer buffer = new StringBuffer();
while (line != null) {