You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/12/07 21:50:43 UTC
[08/14] ambari git commit: Merge branch 'trunk' into
branch-feature-AMBARI-18456
Merge branch 'trunk' into branch-feature-AMBARI-18456
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/276d1244
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/276d1244
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/276d1244
Branch: refs/heads/trunk
Commit: 276d1244e1b1d8200cf644b7cccfaa0a757f8146
Parents: 1f804d1 85c9104
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Nov 29 16:51:58 2016 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 29 17:00:59 2016 -0500
----------------------------------------------------------------------
.../views/stackVersions/stackVersionPage.html | 2 +-
.../org/apache/ambari/logsearch/LogSearch.java | 15 +-
.../logsearch/common/ExternalServerClient.java | 12 +-
.../logsearch/common/LogSearchConstants.java | 1 +
.../AbstractOperationHolderConverter.java | 9 +
...actServiceLogRequestFacetQueryConverter.java | 44 ++
.../BaseServiceLogRequestQueryConverter.java | 1 +
...ServiceLogAnyGraphRequestQueryConverter.java | 13 +
...eLogComponentLevelRequestQueryConverter.java | 15 +-
...eLogComponentRequestFacetQueryConverter.java | 15 +-
...rviceLogLevelCountRequestQueryConverter.java | 16 +
...eLogLevelDateRangeRequestQueryConverter.java | 16 +-
...erviceLogTreeRequestFacetQueryConverter.java | 17 +-
.../ambari/logsearch/doc/DocConstants.java | 1 +
.../logsearch/graph/GraphDataGenerator.java | 26 +-
.../logsearch/manager/ServiceLogsManager.java | 2 +-
.../request/ServiceLogParamDefinition.java | 6 +
.../request/impl/BaseServiceLogRequest.java | 13 +
.../LogsearchKRBAuthenticationFilter.java | 4 +-
.../src/main/resources/logsearch.properties | 2 +-
.../dashboard/BubbleGraphTableLayoutView.js | 4 +-
.../views/dashboard/ComponentListView.js | 2 +-
.../scripts/views/dashboard/HostListView.js | 14 +-
.../main/webapp/scripts/views/tabs/TreeView.js | 10 +-
.../webapp/templates/tabs/TreeView_tmpl.html | 115 ++--
...ComponentLevelRequestQueryConverterTest.java | 7 +-
...ComponentRequestFacetQueryConverterTest.java | 4 +-
.../conf/unix/ambari-metrics-grafana | 34 +-
.../ambari/server/checks/CheckDescription.java | 13 +
.../server/checks/ServicePresenceCheck.java | 177 ++++++
.../internal/ClusterControllerImpl.java | 2 +-
.../ServiceConfigVersionResourceProvider.java | 3 +-
.../logging/LoggingSearchPropertyProvider.java | 11 +-
.../ambari/server/orm/dao/ClusterDAO.java | 24 +
.../ambari/server/orm/dao/ServiceConfigDAO.java | 13 +-
.../orm/entities/ClusterConfigEntity.java | 7 +-
.../entities/ClusterConfigMappingEntity.java | 6 +
.../orm/entities/ServiceConfigEntity.java | 5 +-
.../server/orm/helpers/dbms/MySqlHelper.java | 18 +
.../upgrades/RangerKmsProxyConfig.java | 94 +++
.../server/state/cluster/ClusterImpl.java | 42 +-
.../server/upgrade/SchemaUpgradeHelper.java | 35 +-
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 +
.../FLUME/1.4.0.2.0/package/scripts/flume.py | 1 +
.../package/files/hbaseSmokeVerify.sh | 2 +-
.../HDFS/2.1.0.2.0/kerberos.json | 2 +-
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 19 +
.../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 10 -
.../configuration/logfeeder-ambari-config.xml | 37 ++
.../configuration/logfeeder-output-config.xml | 37 ++
.../configuration/logsearch-properties.xml | 40 ++
.../LOGSEARCH/0.5.0/metainfo.xml | 5 +
.../LOGSEARCH/0.5.0/package/scripts/params.py | 17 +-
.../0.5.0/package/scripts/setup_logfeeder.py | 10 +
.../templates/input.config-ambari.json.j2 | 602 -------------------
.../package/templates/output.config.json.j2 | 61 --
.../properties/input.config-ambari.json.j2 | 602 +++++++++++++++++++
.../0.5.0/properties/output.config.json.j2 | 61 ++
.../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 19 +
.../0.6.0.2.5/package/scripts/master.py | 15 +-
.../0.6.0.2.5/package/scripts/params.py | 28 +-
.../ZEPPELIN/0.6.0.2.5/role_command_order.json | 2 +-
.../HDP/2.0.6/configuration/cluster-env.xml | 6 +
.../HDP/2.2/services/HIVE/themes/theme.json | 107 +++-
.../services/HIVE/configuration/hive-env.xml | 35 ++
.../services/OOZIE/configuration/oozie-env.xml | 33 +
.../stacks/HDP/2.3/services/OOZIE/metainfo.xml | 6 +
.../HDP/2.3/services/OOZIE/themes/theme.json | 116 ++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 7 +
.../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 7 +
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 11 +
.../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 11 +
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 7 +
.../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 7 +
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 11 +
.../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 11 +
.../stacks/HDP/2.5/services/HDFS/kerberos.json | 2 +-
.../stacks/HDP/2.5/services/OOZIE/metainfo.xml | 6 +
.../HDP/2.5/services/OOZIE/themes/theme.json | 116 ++++
.../HDP/2.5/upgrades/host-ordered-upgrade.xml | 11 +-
.../server/checks/ServicePresenceCheckTest.java | 217 +++++++
.../upgrades/RangerKmsProxyConfigTest.java | 125 ++++
.../stacks/2.0.6/HIVE/test_webhcat_server.py | 4 -
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 14 +-
.../test/python/stacks/2.4/configs/default.json | 6 +
ambari-web/app/app.js | 4 -
.../main/admin/service_auto_start.js | 2 +-
.../controllers/main/service/add_controller.js | 11 +-
.../data/configs/services/hive_properties.js | 4 -
.../configs/services/logsearch_properties.js | 51 +-
.../data/configs/services/oozie_properties.js | 4 -
ambari-web/app/messages.js | 3 +
ambari-web/app/styles/application.less | 10 +-
ambari-web/app/styles/bootstrap_overrides.less | 9 +
.../app/styles/enhanced_service_dashboard.less | 343 ++++-------
ambari-web/app/styles/hosts.less | 3 -
.../app/styles/theme/bootstrap-ambari.css | 5 +-
.../modal_popups/widget_browser_popup.hbs | 108 ++--
.../templates/main/admin/service_auto_start.hbs | 2 +-
.../main/service/widgets/create/expression.hbs | 36 +-
.../main/service/widgets/create/step1.hbs | 18 +-
.../main/service/widgets/create/step2.hbs | 22 +-
.../service/widgets/create/step2_add_metric.hbs | 12 +-
.../main/service/widgets/create/step2_graph.hbs | 4 +-
.../service/widgets/create/step2_number.hbs | 4 +-
.../service/widgets/create/step2_template.hbs | 4 +-
.../main/service/widgets/create/step3.hbs | 42 +-
.../create/widget_property_threshold.hbs | 18 +-
ambari-web/app/views/common/chosen_plugin.js | 21 +-
ambari-web/app/views/common/controls_view.js | 2 +-
ambari-web/app/views/main/dashboard/widgets.js | 17 +-
.../main/service/widgets/create/step2_view.js | 2 +
.../main/service/add_controller_test.js | 29 +-
.../test/views/main/dashboard/widgets_test.js | 11 +-
contrib/utils/perf/deploy-gce-perf-cluster.py | 243 +++++---
docs/pom.xml | 4 +-
116 files changed, 2957 insertions(+), 1408 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
index 0000000,bb88f55..25387cc
mode 000000,100644..100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
@@@ -1,0 -1,95 +1,94 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.ambari.server.serveraction.upgrades;
+
+ import java.text.MessageFormat;
+ import java.util.Map;
+ import java.util.concurrent.ConcurrentMap;
+
+ import org.apache.ambari.server.AmbariException;
+ import org.apache.ambari.server.actionmanager.HostRoleStatus;
+ import org.apache.ambari.server.agent.CommandReport;
+ import org.apache.ambari.server.serveraction.AbstractServerAction;
+ import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.Clusters;
+ import org.apache.ambari.server.state.Config;
+ import org.apache.ambari.server.state.SecurityType;
-import org.apache.commons.lang.StringUtils;
+
+ import com.google.inject.Inject;
+
+ /**
+ * Computes Ranger KMS Proxy properties in kms-site
+ */
+
+ public class RangerKmsProxyConfig extends AbstractServerAction {
+ private static final String RANGER_ENV_CONFIG_TYPE = "ranger-env";
+ private static final String RANGER_KMS_SITE_CONFIG_TYPE = "kms-site";
+
+ @Inject
+ private Clusters m_clusters;
+
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+ String clusterName = getExecutionCommand().getClusterName();
+ Cluster cluster = m_clusters.getCluster(clusterName);
+ String outputMsg = "";
+
+ Config rangerEnv = cluster.getDesiredConfigByType(RANGER_ENV_CONFIG_TYPE);
+
+ if (null == rangerEnv) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ MessageFormat.format("Config source type {0} not found, skipping adding properties to {1}.", RANGER_ENV_CONFIG_TYPE, RANGER_KMS_SITE_CONFIG_TYPE), "");
+ }
+
+ String rangerUserProp = "ranger_user";
+ String rangerUser = rangerEnv.getProperties().get(rangerUserProp);
+
+ if (null == rangerUser) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ MessageFormat.format("Required user service user value from {0}/{1} not found, skipping adding properties to {2}.", RANGER_ENV_CONFIG_TYPE, rangerUserProp, RANGER_KMS_SITE_CONFIG_TYPE), "");
+ }
+
+ Config kmsSite = cluster.getDesiredConfigByType(RANGER_KMS_SITE_CONFIG_TYPE);
+
+ if (null == kmsSite) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ MessageFormat.format("Config type {0} not found, skipping adding properties to it.", RANGER_KMS_SITE_CONFIG_TYPE), "");
+ }
+
+ Map<String, String> targetValues = kmsSite.getProperties();
+ if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+ String userProp = "hadoop.kms.proxyuser." + rangerUser + ".users";
+ String groupProp = "hadoop.kms.proxyuser." + rangerUser + ".groups";
+ String hostProp = "hadoop.kms.proxyuser." + rangerUser + ".hosts";
+ targetValues.put(userProp, "*");
+ targetValues.put(groupProp, "*");
+ targetValues.put(hostProp, "*");
+ kmsSite.setProperties(targetValues);
- kmsSite.persist(false);
++ kmsSite.save();
+ outputMsg = outputMsg + MessageFormat.format("Successfully added properties to {0}", RANGER_KMS_SITE_CONFIG_TYPE);
+ } else {
+ outputMsg = outputMsg + MessageFormat.format("Kerberos not enable, not setting proxy properties to {0}", RANGER_KMS_SITE_CONFIG_TYPE);
+ }
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", outputMsg, "");
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
index 0000000,e000c65..7a0d66f
mode 000000,100644..100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
@@@ -1,0 -1,141 +1,125 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.ambari.server.serveraction.upgrades;
+
+ import static org.easymock.EasyMock.anyObject;
+ import static org.easymock.EasyMock.expect;
+ import static org.easymock.EasyMock.replay;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertTrue;
+
+ import java.lang.reflect.Field;
+ import java.util.HashMap;
+ import java.util.Map;
+
+ import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+ import org.apache.ambari.server.actionmanager.HostRoleCommand;
+ import org.apache.ambari.server.agent.CommandReport;
+ import org.apache.ambari.server.agent.ExecutionCommand;
+ import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.SecurityType;
+ import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.ConfigImpl;
++import org.apache.ambari.server.state.SecurityType;
+ import org.easymock.EasyMock;
+ import org.junit.Before;
+ import org.junit.Test;
+
+ import com.google.inject.Injector;
+
+
+ public class RangerKmsProxyConfigTest {
+ private Injector m_injector;
+ private Clusters m_clusters;
+ private Field m_clusterField;
+
+ @Before
+ public void setup() throws Exception {
+ m_injector = EasyMock.createMock(Injector.class);
+ m_clusters = EasyMock.createMock(Clusters.class);
+ Cluster cluster = EasyMock.createMock(Cluster.class);
+
- Config rangerEnv = new ConfigImpl("ranger-env") {
- Map<String, String> mockProperties = new HashMap<String, String>() {{
++ Map<String, String> mockProperties = new HashMap<String, String>() {
++ {
+ put("ranger_user", "ranger");
- }};
-
- @Override
- public Map<String, String> getProperties() {
- return mockProperties;
+ }
+ };
+
- Config kmsSite = new ConfigImpl("kms-site") {
- Map<String, String> mockProperties = new HashMap<String, String>();
- @Override
- public Map<String, String> getProperties() {
- return mockProperties;
- }
-
- @Override
- public void setProperties(Map<String, String> properties) {
- mockProperties.putAll(properties);
- }
++ Config rangerEnv = EasyMock.createNiceMock(Config.class);
++ expect(rangerEnv.getType()).andReturn("ranger-env").anyTimes();
++ expect(rangerEnv.getProperties()).andReturn(mockProperties).anyTimes();
+
- @Override
- public void persist(boolean newConfig) {
- // no-op
- }
- };
++ Config kmsSite = EasyMock.createNiceMock(Config.class);
++ expect(kmsSite.getType()).andReturn("kms-site").anyTimes();
++ expect(kmsSite.getProperties()).andReturn(mockProperties).anyTimes();
+
+ expect(cluster.getDesiredConfigByType("ranger-env")).andReturn(rangerEnv).atLeastOnce();
+ expect(cluster.getDesiredConfigByType("kms-site")).andReturn(kmsSite).atLeastOnce();
+ expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+ expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce();
+ expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).anyTimes();
+
- replay(m_injector, m_clusters, cluster);
++ replay(m_injector, m_clusters, cluster, rangerEnv, kmsSite);
+
+ m_clusterField = RangerKmsProxyConfig.class.getDeclaredField("m_clusters");
+ m_clusterField.setAccessible(true);
+ }
+
+ @Test
+ public void testAction() throws Exception {
+
+ Map<String, String> commandParams = new HashMap<String, String>();
+ commandParams.put("clusterName", "c1");
+
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName("c1");
+
+ HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+ expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+ expect(hrc.getStageId()).andReturn(2L).anyTimes();
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+ replay(hrc);
+
+ RangerKmsProxyConfig action = new RangerKmsProxyConfig();
+ m_clusterField.set(action, m_clusters);
+
+ action.setExecutionCommand(executionCommand);
+ action.setHostRoleCommand(hrc);
+
+ CommandReport report = action.execute(null);
+ assertNotNull(report);
+
+ Cluster c = m_clusters.getCluster("c1");
+ Config config = c.getDesiredConfigByType("kms-site");
+ Map<String, String> map = config.getProperties();
+
+ assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.users"));
+ assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.groups"));
+ assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.hosts"));
+
+
+ assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.users"));
+ assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.groups"));
+ assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.hosts"));
+
+ report = action.execute(null);
+ assertNotNull(report);
+
+ }
+ }